<jupyter_start><jupyter_text>Fill unknown age<jupyter_code>individual_df = full_individual_df
# missing_val = 'not reported/missing'
age_cols = ['geo1_zw2012', 'urban', 'persons', 'sex', 'marst', 'citizen', 'race', 'disabled', 'economic_status']
X = pd.get_dummies(individual_df[age_cols], drop_first=True)

X_train = X[individual_df['age'].notnull()]
gb = GradientBoostingRegressor(n_estimators=100, max_depth=3, random_state=1029)
# rf = RandomForestRegressor(n_estimators=100, max_depth=3, random_state=1029, n_jobs=50)

frac = 0.05
X_tr = X_train.sample(frac=frac, random_state=1029)
gb.fit(X_tr, individual_df.loc[X_tr.index, 'age'])

missing_age = X[individual_df['age'].isnull()]
pd.Series(gb.predict(missing_age)).hist()
X_tr.shape
individual_df.loc[missing_age.index, 'age'] = gb.predict(missing_age).astype(int)
individual_df.loc[missing_age.index].head()
individual_df.economic_status.astype(str).value_counts()<jupyter_output><empty_output><jupyter_text>Create `district_id`, `household_id` and `economic_activity_location_id`<jupyter_code>relevant_cols = [
    'person_id', 'age', 'sex',
    'household_id', 'district_id',
    'economic_status', 'economic_activity_location_id',
    # 'household', 'district'
]
old_new_districts = pd.read_csv(
    get_data_dir('raw', 'district_relation.csv', index_col=0))
old_new_districts.index = old_new_districts['ADMIN_NAME'].str.lower()
individual_df['serial_expanded'] = individual_df['serial']
individual_df['household_id'] = individual_df['serial_expanded'].map(lambda x: f'h_{x}')
individual_df['old_district_id'] = individual_df['geo2_zw2012'].map(lambda x: f'd_{old_new_districts["DIST2012"][x]}')                                          
individual_df['new_district_id'] = individual_df['geo2_zw2012'].map(lambda x: f'd_{old_new_districts["NEW_DIST_ID_2"][x]}')
expanded_individual_df = individual_df.copy()
expanded_individual_df.head()
expanded_individual_df['school_goers'] = 1 * (expanded_individual_df['school_goers'] != 0)
expanded_individual_df.columns
expanded_individual_df.groupby('serial')['sex'].count().mean()
expanded_individual_df.shape[0] / expanded_individual_df['serial'].nunique()
(expanded_individual_df['school_goers'] == 1).sum()
(expanded_individual_df['school_goers'] == 1).sum()<jupyter_output><empty_output><jupyter_text>Mining<jupyter_code># relevant_cols = [
#     'person_id', 'age', 'sex',
#     'household_id', 'district_id',
#     'economic_status', 'economic_activity_location_id',
#     'school_id_district', 
#     'manufacturing_workers',
#     'mining_district_id'
#     # 'household', 'district'
# ]

relevant_cols = [
    'person_id', 'age', 'sex',
    'household_id', 'district_id',
    'economic_status', 'economic_activity_location_id',
    # 'school_id_district', 
    'school_goers',
    'manufacturing_workers',
    # 'mining_district_id'
    # 'household', 'district'
]
mining_df = expanded_individual_df.copy()

p = mining_df['economic_status']

mining_df.loc[p == 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p == 'Disabled and not working', 'household_id']
mining_df.loc[p != 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p != 'Disabled and not working', 'new_district_id']

try:
    mining_df.drop('person_id', axis=1, inplace=True)
except KeyError:
    pass

mining_df['person_id'] = mining_df.index
mining_df['age'] = mining_df['age'].astype(int)
mining_df['economic_status'] = mining_df['economic_status'].str.strip()
mining_df.rename(columns={'new_district_id': 'district_id'}, inplace=True)

mining_df[relevant_cols].to_pickle(
    get_data_dir('preprocessed', 'census', 'zimbabwe_ipums_mining_manufacturing_school_new_dist_100pct.pickle'))
mining_df[relevant_cols].head()<jupyter_output><empty_output><jupyter_text>Contingent on available data<jupyter_code># mining_df['serial_expanded'] = mining_df['serial'].map(lambda x: f'{x}_01')
# mining_df['household_id'] = mining_df['serial_expanded'].map(lambda x: f'h_{x}')
mining_df.loc[mining_df['mining_district_id'] != '', 'mining_district_id'] = mining_df.loc[mining_df['mining_district_id'] != '', 'mining_district_id'].map(lambda x: f'mining_{x}')
mining_df.loc[mining_df['school_id_district'] != '', 'school_id_district'] = mining_df.loc[mining_df['school_id_district'] != '', 'school_id_district'].map(lambda x: f'school_{x}')
mining_df.head()
p = mining_df['economic_status']

mining_df.loc[p == 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p == 'Disabled and not working', 'household_id']
mining_df.loc[p != 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p != 'Disabled and not working', 'new_district_id']
try:
    mining_df.drop('person_id', axis=1, inplace=True)
except KeyError:
    pass

mining_df['person_id'] = mining_df.index
mining_df['age'] = mining_df['age'].astype(int)
mining_df['economic_status'] = mining_df['economic_status'].str.strip()
mining_df.rename(columns={'new_district_id': 'district_id'}, inplace=True)
mining_df[relevant_cols][
    mining_df.serial_expanded.str.endswith('_01')
].to_pickle('../../data/zimbabwe_ipums_mining_manufacturing_school_new_dist_5pct.pickle')
mining_df[relevant_cols][
    mining_df.serial_expanded.str.endswith('_01') | mining_df.serial_expanded.str.endswith('_02')
].to_pickle('../../data/zimbabwe_ipums_mining_manufacturing_school_new_dist_10pct.pickle')
mining_df.shape
import pandas as pd
mining_df = pd.read_pickle('../../data/zimbabwe_ipums_mining_manufacturing_school_new_dist_10pct.pickle')
mining_df[mining_df['school_id_district'] != '']['person_id']<jupyter_output><empty_output><jupyter_text>Expand to 10%<jupyter_code>mining_df<jupyter_output><empty_output><jupyter_text>Scratch<jupyter_code>mining_df[~mining_df['household_id'].str.startswith('h_')]['person_id'].values
mining_df[mining_df.manufacturing_workers.notnull()].person_id
mining_df[mining_df['manufacturing_workers'].notnull()]['person_id'].values
mining_df[mining_df['school_id_district'] != '']['person_id'].values
mining_df['school_id_district'].shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15878212473399902, 'fraction_numerical': 0.01162219675888034, 'mean_word_length': 8.652448657187994, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '404960', 'license': 'MIT', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2189, 'n_words': 262, 'path': 'notebooks/census/Zimbabwe Raw Data Full Simulated School Mining Manufacturing.ipynb', 'repo_name': 'pawan100/covid19-agent-based-model', 'size': '6109'}