<commit_before>"""

Every child class of this Job class is launchable via a jenkins job on the computing server.

"""


import os
from datetime import datetime
import logging
from labonneboite.importer import settings
from labonneboite.importer import util as import_util
from labonneboite.common.util import timeit
from labonneboite.importer.models.computing import ImportTask

logger = logging.getLogger('main')


class Job(object):
    file_type = None
    table_name = None
    import_type = None
    input_filename = None

    @timeit
    def run(self):
        if self.file_type:
            logger.info("file type:%s, checking the task is runnable with that file", self.file_type)
            self.check_runnable()
        if settings.BACKUP_FIRST:
            logger.info("backing up existing table before proceeding...")
            self.back_up_input_table()
        if self.import_type:
            logger.info("recording in db we processed this input file...")
            self.record_task()
        logger.info("let's run this task!")
        self.run_task()
        self.after_check()
        logger.info("task is done.")

    def check_runnable(self):
        files = [
            os.path.join(os.path.normpath(settings.INPUT_SOURCE_FOLDER), name)
                for name in os.listdir(settings.INPUT_SOURCE_FOLDER)
        ]
        # We only compare basename here, to solve a bug met in jenkins environment only.
        # In jenkins environment:
        # /srv/lbb is a symlink to /srv/jenkins/workspace/lbb
        # so that /srv/lbb/labonneboite path is the same on both local test and jenkins test.
        # However here,
        # self.input_filename == /srv/jenkins/workspace/lbb/labonneboite/importer/tests/data/...
        # but files are like /srv/lbb/labonneboite/importer/tests/data/...
        # hence why we use basename here.
        if not os.path.basename(self.input_filename) in [os.path.basename(f) for f in files]:
            msg = "File %s does not exist: not found in list %s" % (
                self.input_filename, ','.join(files))
            raise Exception(msg)
        if import_util.is_processed(self.input_filename):
            raise "fatal error : cannot run task %s which is already marked as processed" % self.input_filename

    def after_check(self):
        pass

    def record_task(self):
        task = ImportTask(
            filename=os.path.basename(self.input_filename),
            state=ImportTask.FILE_READ,
            import_type=self.import_type,
        )
        task.save()

    @timeit
    def back_up_input_table(self):
        timestamp = datetime.now().strftime('%Y_%m_%d_%H%M')
        import_util.back_up(settings.BACKUP_INPUT_FOLDER, self.table_name, self.file_type, timestamp)
<commit_msg>use realpath instead of basename<commit_after>"""

Every child class of this Job class is launchable via a jenkins job on the computing server.

"""


import os
from datetime import datetime
import logging
from labonneboite.importer import settings
from labonneboite.importer import util as import_util
from labonneboite.common.util import timeit
from labonneboite.importer.models.computing import ImportTask

logger = logging.getLogger('main')


class Job(object):
    file_type = None
    table_name = None
    import_type = None
    input_filename = None

    @timeit
    def run(self):
        if self.file_type:
            logger.info("file type:%s, checking the task is runnable with that file", self.file_type)
            self.check_runnable()
        if settings.BACKUP_FIRST:
            logger.info("backing up existing table before proceeding...")
            self.back_up_input_table()
        if self.import_type:
            logger.info("recording in db we processed this input file...")
            self.record_task()
        logger.info("let's run this task!")
        self.run_task()
        self.after_check()
        logger.info("task is done.")

    def check_runnable(self):
        files = [
            os.path.join(os.path.normpath(settings.INPUT_SOURCE_FOLDER), name)
                for name in os.listdir(settings.INPUT_SOURCE_FOLDER)
        ]
        # We only compare realpath here, to solve a bug met in jenkins environment only.
        # In jenkins environment:
        # /srv/lbb is a symlink to /srv/jenkins/workspace/lbb
        # so that /srv/lbb/labonneboite path is the same on both local test and jenkins test.
        # However here,
        # self.input_filename == /srv/jenkins/workspace/lbb/labonneboite/importer/tests/data/...
        # but files are like /srv/lbb/labonneboite/importer/tests/data/...
        # hence why we use realpath here.
        if not os.path.realpath(self.input_filename) in [os.path.realpath(f) for f in files]:
            msg = "File %s does not exist: not found in list %s" % (
                self.input_filename, ','.join(files))
            raise Exception(msg)
        if import_util.is_processed(self.input_filename):
            raise "fatal error : cannot run task %s which is already marked as processed" % self.input_filename

    def after_check(self):
        pass

    def record_task(self):
        task = ImportTask(
            filename=os.path.basename(self.input_filename),
            state=ImportTask.FILE_READ,
            import_type=self.import_type,
        )
        task.save()

    @timeit
    def back_up_input_table(self):
        timestamp = datetime.now().strftime('%Y_%m_%d_%H%M')
        import_util.back_up(settings.BACKUP_INPUT_FOLDER, self.table_name, self.file_type, timestamp)

{'fraction_non_alphanumeric': 0.09206977162380867, 'fraction_numerical': 0.0, 'mean_word_length': 2.5224825839138694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3609690', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1652, 'n_words': 452}