<commit_before>"""
Extracts data from CSV file into a MySQL table.

It is assumed the extract function will not be called twice for the same file
(otherwise duplicate records will be created). This should be taken care of by the caller.

For each extraction, we record its extraction datetime and its most recent dpae date.
"""
import re
import time
from datetime import datetime
import sys
from sqlalchemy.exc import OperationalError
import os

from labonneboite.importer import settings
from labonneboite.importer import util as import_util
from labonneboite.common.util import timeit
from labonneboite.importer.util import parse_dpae_line, InvalidRowException, history_importer_job_decorator
from labonneboite.importer.models.computing import DpaeStatistics, ImportTask, Hiring
from labonneboite.importer.jobs.base import Job
from labonneboite.importer.jobs.common import logger
from labonneboite.common.database import db_session


class DpaeExtractJob(Job):
    file_type = "dpae"
    import_type = ImportTask.DPAE
    table_name = settings.HIRING_TABLE

    def __init__(self, filename):
        self.input_filename = filename
        self.last_historical_data_date_in_file = None
        self.zipcode_errors = 0
        self.invalid_row_errors = 0

    @timeit
    def run_task(self):<commit_msg>fix file_type<commit_after>"""
Extracts data from CSV file into a MySQL table.

It is assumed the extract function will not be called twice for the same file
(otherwise duplicate records will be created). This should be taken care of by the caller.

For each extraction, we record its extraction datetime and its most recent dpae date.
"""
import re
import time
from datetime import datetime
import sys
from sqlalchemy.exc import OperationalError
import os

from labonneboite.importer import settings
from labonneboite.importer import util as import_util
from labonneboite.common.util import timeit
from labonneboite.importer.util import parse_dpae_line, InvalidRowException, history_importer_job_decorator
from labonneboite.importer.models.computing import DpaeStatistics, ImportTask, Hiring
from labonneboite.importer.jobs.base import Job
from labonneboite.importer.jobs.common import logger
from labonneboite.common.database import db_session


class DpaeExtractJob(Job):
    file_type = DpaeStatistics.DPAE
    import_type = ImportTask.DPAE
    table_name = settings.HIRING_TABLE

    def __init__(self, filename):
        self.input_filename = filename
        self.last_historical_data_date_in_file = None
        self.zipcode_errors = 0
        self.invalid_row_errors = 0

    @timeit
    def run_task(self):
{'fraction_non_alphanumeric': 0.048491790759831996, 'fraction_numerical': 0.0015273004963726614, 'mean_word_length': 4.708061002178649, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4030828', 'n_tokens_mistral': 771, 'n_tokens_neox': 741, 'n_words': 260}