<commit_before>import luigi
import luigi.mock
import numpy as np
import pandas as pd
from sklearn import preprocessing

from clean_data import ImputeSchoolsMissingData, ImputeTeacherMissingData
from mapeamentos import dicionario_questoes_nomes_escola


class EncodeSchoolQuestions(luigi.Task):
    input_task = ImputeSchoolsMissingData()
    mappings = {'A': 3, 'B': 2, 'C': 1, 'D': 0}

    def requires(self):
        return self.input_task

    def output(self):
        return luigi.mock.MockTarget('./dados/2013/TS_ESCOLA_with_encoded_values.csv')

    def run(self):
        with self.input_task.output().open('r') as fp:
            escolas_2013_pd = pd.read_csv(fp)

        for col in escolas_2013_pd.filter(regex='TX_RESP_.*'):
            escolas_2013_pd[col].replace(to_replace=self.mappings, inplace=True)
        escolas_2013_pd.NIVEL_SOCIO_ECONOMICO.replace(to_replace={'Grupo {}'.format(i + 1): i + 1 for i in range(8)},
                                                      inplace=True)

        with self.output().open('w') as fp:
            escolas_2013_pd.to_csv(fp, index=False)


class EncodeTeacherQuestions(luigi.Task):
    input_task = ImputeTeacherMissingData()
    mappings = {'A': 0, 'B': 1, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'H': 7, 'I': 8, 'J': 9, 'K': 10}

    def requires(self):
        return self.input_task

    def output(self):
        return luigi.mock.MockTarget('./dados/2013/TS_PROFESSOR_with_encoded_values.csv')
<commit_msg>Persiste dados antes do escalonamento e apos codificacao<commit_after>import luigi
import luigi.mock
import numpy as np
import pandas as pd
from sklearn import preprocessing

from clean_data import ImputeSchoolsMissingData, ImputeTeacherMissingData
from mapeamentos import dicionario_questoes_nomes_escola


class EncodeSchoolQuestions(luigi.Task):
    input_task = ImputeSchoolsMissingData()
    mappings = {'A': 3, 'B': 2, 'C': 1, 'D': 0}

    def requires(self):
        return self.input_task

    def output(self):
        return luigi.LocalTarget('./dados/2013/TS_ESCOLA_with_encoded_values.csv')

    def run(self):
        with self.input_task.output().open('r') as fp:
            escolas_2013_pd = pd.read_csv(fp)

        for col in escolas_2013_pd.filter(regex='TX_RESP_.*'):
            escolas_2013_pd[col].replace(to_replace=self.mappings, inplace=True)
        escolas_2013_pd.NIVEL_SOCIO_ECONOMICO.replace(to_replace={'Grupo {}'.format(i + 1): i + 1 for i in range(8)},
                                                      inplace=True)

        with self.output().open('w') as fp:
            escolas_2013_pd.to_csv(fp, index=False)


class EncodeTeacherQuestions(luigi.Task):
    input_task = ImputeTeacherMissingData()
    mappings = {'A': 0, 'B': 1, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'H': 7, 'I': 8, 'J': 9, 'K': 10}

    def requires(self):
        return self.input_task

    def output(self):
        return luigi.LocalTarget('./dados/2013/TS_PROFESSOR_with_encoded_values.csv')

{'fraction_non_alphanumeric': 0.12596314907872697, 'fraction_numerical': 0.03149078726968174, 'mean_word_length': 3.193820224719101, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '967564', 'n_tokens_mistral': 1126, 'n_tokens_neox': 1015, 'n_words': 229}