<commit_before>from __future__ import unicode_literals
import os
import time
import json
import sys
import shutil
from converter import Converter
from extensions import valid_input_extensions, valid_output_extensions, bad_subtitle_codecs, valid_subtitle_extensions
from qtfaststart import processor, exceptions
from babelfish import Language


class MkvtoMp4:
    def __init__(   self, settings=None, 
                    FFMPEG_PATH="FFMPEG.exe", 
                    FFPROBE_PATH="FFPROBE.exe", 
                    delete=True, 
                    output_extension='mp4', 
                    output_dir=None, 
                    relocate_moov=True, 
                    output_format = 'mp4', 
                    video_codec=['h264', 'x264'], 
                    audio_codec=['ac3'], 
                    audio_bitrate=None, 
                    iOS=False, 
                    awl=None, 
                    swl=None, 
                    adl=None, 
                    sdl=None, 
                    downloadsubs=True,
                    processMP4=False, 
                    copyto=None, 
                    moveto=None,
                    embedsubs=True,
                    providers=['addic7ed', 'podnapisi', 'thesubdb', 'opensubtitles']):
        # Settings
        self.FFMPEG_PATH=FFMPEG_PATH
        self.FFPROBE_PATH=FFPROBE_PATH
        self.delete=delete
        self.output_extension=output_extension
        self.output_format=output_format
        self.output_dir=output_dir
        self.relocate_moov=relocate_moov
        self.processMP4=processMP4
        self.copyto=copyto
        self.moveto=moveto
        self.relocate_moov=relocate_moov
        # Video settings
        self.video_codec=video_codec
        # Audio settings
        self.audio_codec=audio_codec
        self.audio_bitrate=audio_bitrate
        self.iOS=iOS
        self.awl=awl
        self.adl=adl
        # Subtitle settings
        self.swl=swl
        self.sdl=sdl
        self.downloadsubs = downloadsubs
        self.subproviders = providers
        self.embedsubs = embedsubs

        # Import settings
        if settings is not None: self.importSettings(settings)
        self.options = None
        self.deletesubs = set()

    def importSettings(self, settings):
        self.FFMPEG_PATH=settings.ffmpeg
        self.FFPROBE_PATH=settings.ffprobe
        self.delete=settings.delete
        self.output_extension=settings.output_extension
        self.output_format=settings.output_format
        self.output_dir=settings.output_dir
        self.relocate_moov=settings.relocate_moov
        self.processMP4=settings.processMP4
        self.copyto=settings.copyto
        self.moveto=settings.moveto
        self.relocate_moov = settings.relocate_moov
        #Video settings
        self.video_codec=settings.vcodec
        #Audio settings
        self.audio_codec=settings.acodec
        #self.audio_bitrate=settings.abitrate
        self.iOS=settings.iOS
        self.awl=settings.awl
        self.adl=settings.adl
        #Subtitle settings
        self.swl=settings.swl
        self.sdl=settings.sdl
        self.downloadsubs=settings.downloadsubs
        self.subproviders=settings.subproviders
        self.embedsubs=settings.embedsubs

    # Process a file from start to finish, with checking to make sure formats are compatible with selected settings
    def process(self, inputfile, reportProgress=False, original=None):
        delete = self.delete
        deleted = False
        options = None
        if not self.validSource(inputfile): return False

        if self.needProcessing(inputfile):
            options = self.generateOptions(inputfile, original=original)
            if reportProgress: print json.dumps(options, sort_keys=False, indent=4)
            outputfile, inputfile = self.convert(inputfile, options, reportProgress)
            if not outputfile: return False
        else:
            outputfile = inputfile
            if self.output_dir is not None:
                try:                
                    outputfile = os.path.join(self.output_dir, os.path.split(inputfile)[1])
                    shutil.copy(inputfile, outputfile)
                except Exception as e:
                    print "Error moving file to output directory"
                    print e
                    delete = False
            else:
                delete = False



        if delete:
            if self.removeFile(inputfile):
                print inputfile + " deleted"
                deleted = True
            else:
                print "Couldn't delete the original file:" + inputfile
            if self.downloadsubs:
                for subfile in self.deletesubs:
                    if self.removeFile(subfile):
                        print subfile + "deleted"

        dim = self.getDimensions(outputfile)

        return { 'input': inputfile,
                 'output': outputfile,<commit_msg>Catch special character errors<commit_after>from __future__ import unicode_literals
import os
import time
import json
import sys
import shutil
from converter import Converter
from extensions import valid_input_extensions, valid_output_extensions, bad_subtitle_codecs, valid_subtitle_extensions
from qtfaststart import processor, exceptions
from babelfish import Language


class MkvtoMp4:
    def __init__(   self, settings=None, 
                    FFMPEG_PATH="FFMPEG.exe", 
                    FFPROBE_PATH="FFPROBE.exe", 
                    delete=True, 
                    output_extension='mp4', 
                    output_dir=None, 
                    relocate_moov=True, 
                    output_format = 'mp4', 
                    video_codec=['h264', 'x264'], 
                    audio_codec=['ac3'], 
                    audio_bitrate=None, 
                    iOS=False, 
                    awl=None, 
                    swl=None, 
                    adl=None, 
                    sdl=None, 
                    downloadsubs=True,
                    processMP4=False, 
                    copyto=None, 
                    moveto=None,
                    embedsubs=True,
                    providers=['addic7ed', 'podnapisi', 'thesubdb', 'opensubtitles']):
        # Settings
        self.FFMPEG_PATH=FFMPEG_PATH
        self.FFPROBE_PATH=FFPROBE_PATH
        self.delete=delete
        self.output_extension=output_extension
        self.output_format=output_format
        self.output_dir=output_dir
        self.relocate_moov=relocate_moov
        self.processMP4=processMP4
        self.copyto=copyto
        self.moveto=moveto
        self.relocate_moov=relocate_moov
        # Video settings
        self.video_codec=video_codec
        # Audio settings
        self.audio_codec=audio_codec
        self.audio_bitrate=audio_bitrate
        self.iOS=iOS
        self.awl=awl
        self.adl=adl
        # Subtitle settings
        self.swl=swl
        self.sdl=sdl
        self.downloadsubs = downloadsubs
        self.subproviders = providers
        self.embedsubs = embedsubs

        # Import settings
        if settings is not None: self.importSettings(settings)
        self.options = None
        self.deletesubs = set()

    def importSettings(self, settings):
        self.FFMPEG_PATH=settings.ffmpeg
        self.FFPROBE_PATH=settings.ffprobe
        self.delete=settings.delete
        self.output_extension=settings.output_extension
        self.output_format=settings.output_format
        self.output_dir=settings.output_dir
        self.relocate_moov=settings.relocate_moov
        self.processMP4=settings.processMP4
        self.copyto=settings.copyto
        self.moveto=settings.moveto
        self.relocate_moov = settings.relocate_moov
        #Video settings
        self.video_codec=settings.vcodec
        #Audio settings
        self.audio_codec=settings.acodec
        #self.audio_bitrate=settings.abitrate
        self.iOS=settings.iOS
        self.awl=settings.awl
        self.adl=settings.adl
        #Subtitle settings
        self.swl=settings.swl
        self.sdl=settings.sdl
        self.downloadsubs=settings.downloadsubs
        self.subproviders=settings.subproviders
        self.embedsubs=settings.embedsubs

    # Process a file from start to finish, with checking to make sure formats are compatible with selected settings
    def process(self, inputfile, reportProgress=False, original=None):
        delete = self.delete
        deleted = False
        options = None
        if not self.validSource(inputfile): return False

        if self.needProcessing(inputfile):
            options = self.generateOptions(inputfile, original=original)
            if reportProgress: print json.dumps(options, sort_keys=False, indent=4)
            outputfile, inputfile = self.convert(inputfile, options, reportProgress)
            if not outputfile: return False
        else:
            outputfile = inputfile
            if self.output_dir is not None:
                try:                
                    outputfile = os.path.join(self.output_dir, os.path.split(inputfile)[1])
                    shutil.copy(inputfile, outputfile)
                except Exception as e:
                    print "Error moving file to output directory"
                    print e
                    delete = False
            else:
                delete = False



        if delete:
            if self.removeFile(inputfile):
                try:
                    print inputfile + " deleted"
                except:
                    print "Original file deleted"
                deleted = True
            else:
                print "Couldn't delete the original file:" + inputfile
            if self.downloadsubs:
                for subfile in self.deletesubs:
                    if self.removeFile(subfile):
                        print subfile + "deleted"

        dim = self.getDimensions(outputfile)

        return { 'input': inputfile,
                 'output': outputfile,
{'fraction_non_alphanumeric': 0.06694644284572342, 'fraction_numerical': 0.0035971223021582736, 'mean_word_length': 1.7771920088790234, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1814458', 'n_tokens_mistral': 2782, 'n_tokens_neox': 2594, 'n_words': 536}