<commit_before># coding=utf-8
from __future__ import unicode_literals

import logging

import daiquiri
import os
import traceback

import yaml
from guessit import guessit
import patoolib

import utils as utils


class TvSort(object):
    is_any_error = False
    project_name = 'tvsort_sl'
    settings = dict(PROJECT_NAME=project_name)
    logger = None

    def __init__(self, is_test=False, **kwargs):
        self.base_dir, self.settings_folder = self.get_settings_folders()
        self.load_base_setting()

        if self.check_project_setup(is_test):
            self.load_additional_settings(is_test=is_test)

            self.create_logger(**kwargs)

    def run(self):
        if not utils.is_process_already_running(self.settings.get('DUMMY_FILE_PATH')):
            try:
                utils.create_file(self.settings.get('DUMMY_FILE_PATH'))

                for file_path in utils.get_files(self.settings.get('UNSORTED_PATH')):
                    if utils.is_compressed(file_path, self.settings):
                        self.logger.info("Extracting {}".format(file_path))
                        patoolib.extract_archive(file_path, outdir=self.settings.get('UNSORTED_PATH'), verbosity=-1)
                        utils.delete_file(file_path, self.logger)

                for file_path in utils.get_files(self.settings.get('UNSORTED_PATH')):
                    self.logger.info('Checking file: {}'.format(file_path))

                    if utils.is_garbage_file(file_path, self.settings):
                        self.logger.info('Removing file: {}'.format(file_path))
                        utils.delete_file(file_path, self.logger)
                    elif utils.is_media(file_path, self.settings):
                        guess = guessit(file_path)
                        new_path = None
                        if utils.is_tv_show(guess):
                            base = self.settings.get('TV_PATH')
                            utils.remove_wrong_country_data(guess)
                            show_name = utils.transform_to_path_name(utils.get_show_name(guess))
                            utils.add_missing_country(guess, show_name)
                            if guess.get('country'):
                                show_name += '.{}'.format(guess.get('country'))

                            new_path = os.path.join(base, show_name)
                            utils.create_folder(new_path, self.logger)

                        elif utils.is_movie(guess):
                            new_path = self.settings.get('MOVIES_PATH')

                        utils.copy_file(file_path, new_path, self.logger, move_file=self.settings.get('MOVE_FILES'))

                    folder_path = utils.get_folder_path_from_file_path(file_path)
                    utils.delete_folder_if_empty(folder_path, self.logger)

                # clean up
                for folder_path in utils.get_folders(self.settings.get('UNSORTED_PATH')):
                    utils.delete_folder_if_empty(folder_path, self.logger)

                # download subtitles
                utils.download_subtitles(tv_sort.settings)

                # Update XBMC
                utils.update_xbmc(self.settings.get('KODI_IP'), self.logger)

            except Exception as e:
                utils.is_any_error = True
                self.logger.error(e)
                self.logger.error(traceback.print_exc())

            finally:
                utils.delete_file(self.settings.get('DUMMY_FILE_PATH'), self.logger)
                return not self.is_any_error
        else:
            self.logger.info('Proses already running')
            return False

    def get_settings_folders(self):
        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        settings_folder = os.path.join(base_dir, self.project_name, 'settings')

        return base_dir, settings_folder

    def load_base_setting(self):
        conf_files = self.get_settings_file(self.settings_folder)
        self.update_settings_from_file(conf_files)

        self.build_base_settings()

    def get_settings_file(self, is_base=True, is_test=False):
        if is_base:
            return [os.path.join(self.settings_folder, 'conf.yml')]
        else:
            conf_files = [os.path.join(self.settings_folder, 'local.yml')]
            if is_test:
                conf_files.append(os.path.join(self.settings_folder, 'test.yml'))

            return  conf_files

    def update_settings_from_file(self, conf_files):
        for file_path in conf_files:
            self.settings.update(yaml.load(open(file_path)))

    def build_base_settings(self):
        self.settings['LOG_PATH'] = os.path.join(self.base_dir, 'logs')

    def load_additional_settings(self, is_test=False):
        conf_files = self.get_settings_file(is_base=False, is_test=is_test)
        self.update_settings_from_file(conf_files)

        self.build_settings()

    def build_settings(self):
        # This should be overwrite by prod OR test settings
        self.settings['TV_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'TVShows')
        self.settings['MOVIES_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'Movies')
        self.settings['UNSORTED_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'Unsortted')
        self.settings['DUMMY_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'Dummy')

        self.settings['TEST_FILES'] = os.path.join(self.base_dir, 'tvsort_sl', 'tests', 'test_files')<commit_msg>another minor change<commit_after># coding=utf-8
from __future__ import unicode_literals

import logging

import daiquiri
import os
import traceback

import yaml
from guessit import guessit
import patoolib

import utils as utils


class TvSort(object):
    is_any_error = False
    project_name = 'tvsort_sl'
    settings = dict(PROJECT_NAME=project_name)
    logger = None

    def __init__(self, is_test=False, **kwargs):
        self.base_dir, self.settings_folder = self.get_settings_folders()
        self.load_base_setting()

        if self.check_project_setup(is_test):
            self.load_additional_settings(is_test=is_test)

            self.create_logger(**kwargs)

    def run(self):
        if not utils.is_process_already_running(self.settings.get('DUMMY_FILE_PATH')):
            try:
                utils.create_file(self.settings.get('DUMMY_FILE_PATH'))

                for file_path in utils.get_files(self.settings.get('UNSORTED_PATH')):
                    if utils.is_compressed(file_path, self.settings):
                        self.logger.info("Extracting {}".format(file_path))
                        patoolib.extract_archive(file_path, outdir=self.settings.get('UNSORTED_PATH'), verbosity=-1)
                        utils.delete_file(file_path, self.logger)

                for file_path in utils.get_files(self.settings.get('UNSORTED_PATH')):
                    self.logger.info('Checking file: {}'.format(file_path))

                    if utils.is_garbage_file(file_path, self.settings):
                        self.logger.info('Removing file: {}'.format(file_path))
                        utils.delete_file(file_path, self.logger)
                    elif utils.is_media(file_path, self.settings):
                        guess = guessit(file_path)
                        new_path = None
                        if utils.is_tv_show(guess):
                            base = self.settings.get('TV_PATH')
                            utils.remove_wrong_country_data(guess)
                            show_name = utils.transform_to_path_name(utils.get_show_name(guess))
                            utils.add_missing_country(guess, show_name)
                            if guess.get('country'):
                                show_name += '.{}'.format(guess.get('country'))

                            new_path = os.path.join(base, show_name)
                            utils.create_folder(new_path, self.logger)

                        elif utils.is_movie(guess):
                            new_path = self.settings.get('MOVIES_PATH')

                        utils.copy_file(file_path, new_path, self.logger, move_file=self.settings.get('MOVE_FILES'))

                    folder_path = utils.get_folder_path_from_file_path(file_path)
                    utils.delete_folder_if_empty(folder_path, self.logger)

                # clean up
                for folder_path in utils.get_folders(self.settings.get('UNSORTED_PATH')):
                    utils.delete_folder_if_empty(folder_path, self.logger)

                # download subtitles
                utils.download_subtitles(tv_sort.settings)

                # Update XBMC
                utils.update_xbmc(self.settings.get('KODI_IP'), self.logger)

            except Exception as e:
                utils.is_any_error = True
                self.logger.error(e)
                self.logger.error(traceback.print_exc())

            finally:
                utils.delete_file(self.settings.get('DUMMY_FILE_PATH'), self.logger)
                return not self.is_any_error
        else:
            self.logger.info('Proses already running')
            return False

    def get_settings_folders(self):
        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        settings_folder = os.path.join(base_dir, self.project_name, 'settings')

        return base_dir, settings_folder

    def load_base_setting(self):
        conf_files = self.get_settings_file(self.settings_folder)
        self.update_settings_from_file(conf_files)

        self.build_base_settings()

    def get_settings_file(self, is_base=True, is_test=False):
        if is_base:
            return [os.path.join(self.settings_folder, 'conf.yml')]
        else:
            conf_files = [os.path.join(self.settings_folder, 'local.yml')]
            if is_test:
                conf_files.append(os.path.join(self.settings_folder, 'test.yml'))

            return conf_files

    def update_settings_from_file(self, conf_files):
        for file_path in conf_files:
            self.settings.update(yaml.load(open(file_path)))

    def build_base_settings(self):
        self.settings['LOG_PATH'] = os.path.join(self.base_dir, 'logs')

    def load_additional_settings(self, is_test=False):
        conf_files = self.get_settings_file(is_base=False, is_test=is_test)
        self.update_settings_from_file(conf_files)

        self.build_settings()

    def build_settings(self):
        # This should be overwrite by prod OR test settings
        self.settings['TV_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'TVShows')
        self.settings['MOVIES_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'Movies')
        self.settings['UNSORTED_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'Unsortted')
        self.settings['DUMMY_PATH'] = os.path.join(self.settings.get('BASE_DRIVE'), 'Dummy')

        self.settings['TEST_FILES'] = os.path.join(self.base_dir, 'tvsort_sl', 'tests', 'test_files')
{'fraction_non_alphanumeric': 0.10253623188405797, 'fraction_numerical': 0.00036231884057971015, 'mean_word_length': 2.2077280650784425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2722487', 'n_tokens_mistral': 3355, 'n_tokens_neox': 3233, 'n_words': 483}