<commit_before>import luigi
import os
import pandas as pd
from db import extract
from db import log
from forecast import compute
import shutil
import luigi.contrib.hadoop
from pathlib import Path
from sqlalchemy import create_engine
from pysandag.database import get_connection_string
import pydefm.compute as cp
import numpy as np
from forecast import util
from pydefm import utils


class Population(luigi.Task):
    year = luigi.Parameter()
    dem_id = luigi.Parameter()
    econ_id = luigi.Parameter()

    def requires(self):
        return None

    def output(self):
        return luigi.LocalTarget('temp/data.h5')

    def run(self):

        my_file = Path('temp/data.h5')
        if my_file.is_file():
            print'File exists'
        else:
            db_run_id = log.new_run(dem_id=self.dem_id, econ_id=self.econ_id)
            run_id = pd.Series([db_run_id])
            run_id.to_hdf('temp/data.h5', 'run_id',  mode='a')

            dem_sim_rates = extract.create_df('dem_sim_rates', 'dem_sim_rates_table',
                                              rate_id=self.dem_id, index=None)

            dem_sim_rates.to_hdf('temp/data.h5', 'dem_sim_rates',  mode='a')

            pop = extract.create_df('population', 'population_table', rate_id=dem_sim_rates.base_population_id[0])
            pop.to_hdf('temp/data.h5', 'pop', mode='a')

            pop2 = pop[(pop['type'] == 'HHP')]
            pop2 = pop2.reset_index(drop=False)

            pop2 = pd.DataFrame(pop2['persons'].groupby([pop2['age'], pop2['race_ethn'], pop2['sex']]).sum())
            pop2.rename(columns={'persons': 'persons_sum'}, inplace=True)

            pop2 = pop.join(pop2)
            pop2['rates'] = np.where(pop2['type'].isin(['INS', 'OTH']), (pop2['persons'] / pop2['persons_sum']), 0)

            rates = pop2[['mildep', 'type', 'rates']]
            rates.to_hdf('temp/data.h5', 'ins_oth_rates', mode='a')

            engine = create_engine(get_connection_string("model_config.yml", 'output_database'))
            population_summary = []
            population_summary.append({'Year': self.year - 1,
                                       'Run_id': run_id[0],
                                       'Population': pop['persons'].sum(),
                                       'mig_out': 0,
                                       'mig_in': 0,
                                       'deaths': 0,
                                       'new_born': 0})

            summary_df = pd.DataFrame(population_summary)
            summary_df.to_sql(name='population_summary', con=engine, schema='defm', if_exists='append', index=False)
            pop['yr'] = 2010
            pop['run_id'] = db_run_id
            pop.to_sql(name='population', con=engine, schema='defm', if_exists='append', index=True)


class InMigrationRates(luigi.Task):
    year = luigi.Parameter()
    dem_id = luigi.Parameter()
    econ_id = luigi.Parameter()

    def requires(self):
        return Population(year=self.year, dem_id=self.dem_id, econ_id=self.econ_id)

    def output(self):
        return luigi.LocalTarget('temp/data.h5')

    def run(self):<commit_msg>get base year for simulation (remove hardcoded year)<commit_after>import luigi
import os
import pandas as pd
from db import extract
from db import log
from forecast import compute
import shutil
import luigi.contrib.hadoop
from pathlib import Path
from sqlalchemy import create_engine
from pysandag.database import get_connection_string
import pydefm.compute as cp
import numpy as np
from forecast import util
from pydefm import utils


class Population(luigi.Task):
    year = luigi.Parameter()
    dem_id = luigi.Parameter()
    econ_id = luigi.Parameter()

    def requires(self):
        return None

    def output(self):
        return luigi.LocalTarget('temp/data.h5')

    def run(self):

        my_file = Path('temp/data.h5')
        if my_file.is_file():
            print'File exists'
        else:
            db_run_id = log.new_run(dem_id=self.dem_id, econ_id=self.econ_id)
            run_id = pd.Series([db_run_id])
            run_id.to_hdf('temp/data.h5', 'run_id',  mode='a')

            dem_sim_rates = extract.create_df('dem_sim_rates', 'dem_sim_rates_table',
                                              rate_id=self.dem_id, index=None)

            dem_sim_rates.to_hdf('temp/data.h5', 'dem_sim_rates',  mode='a')

            pop = extract.create_df('population', 'population_table', rate_id=dem_sim_rates.base_population_id[0])
            pop.to_hdf('temp/data.h5', 'pop', mode='a')

            pop2 = pop[(pop['type'] == 'HHP')]
            pop2 = pop2.reset_index(drop=False)

            pop2 = pd.DataFrame(pop2['persons'].groupby([pop2['age'], pop2['race_ethn'], pop2['sex']]).sum())
            pop2.rename(columns={'persons': 'persons_sum'}, inplace=True)

            pop2 = pop.join(pop2)
            pop2['rates'] = np.where(pop2['type'].isin(['INS', 'OTH']), (pop2['persons'] / pop2['persons_sum']), 0)

            rates = pop2[['mildep', 'type', 'rates']]
            rates.to_hdf('temp/data.h5', 'ins_oth_rates', mode='a')

            engine = create_engine(get_connection_string("model_config.yml", 'output_database'))
            population_summary = []
            population_summary.append({'Year': self.year - 1,
                                       'Run_id': run_id[0],
                                       'Population': pop['persons'].sum(),
                                       'mig_out': 0,
                                       'mig_in': 0,
                                       'deaths': 0,
                                       'new_born': 0})

            summary_df = pd.DataFrame(population_summary)
            summary_df.to_sql(name='population_summary', con=engine, schema='defm', if_exists='append', index=False)
            pop['yr'] = self.year - 1
            pop['run_id'] = db_run_id
            pop.to_sql(name='population', con=engine, schema='defm', if_exists='append', index=True)


class InMigrationRates(luigi.Task):
    year = luigi.Parameter()
    dem_id = luigi.Parameter()
    econ_id = luigi.Parameter()

    def requires(self):
        return Population(year=self.year, dem_id=self.dem_id, econ_id=self.econ_id)

    def output(self):
        return luigi.LocalTarget('temp/data.h5')

    def run(self):
{'fraction_non_alphanumeric': 0.13131313131313133, 'fraction_numerical': 0.01057449494949495, 'mean_word_length': 2.2732438016528924, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6834561', 'n_tokens_mistral': 2067, 'n_tokens_neox': 1949, 'n_words': 371}