<jupyter_start><jupyter_text>data cleaning<jupyter_code>app_data['CODE_GENDER'].replace('XNA', np.nan, inplace=True)
app_data['DAYS_EMPLOYED'].replace(365243, np.nan, inplace=True)
app_data['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True)
app_data['NAME_FAMILY_STATUS'].replace('Unknown', np.nan, inplace=True)
app_data['ORGANIZATION_TYPE'].replace('XNA', np.nan, inplace=True)<jupyter_output><empty_output><jupyter_text>HandCrafted Features<jupyter_code>app_data['annuity_income_percentage'] = app_data['AMT_ANNUITY'] / app_data['AMT_INCOME_TOTAL']
app_data['car_to_birth_ratio'] = app_data['OWN_CAR_AGE'] / app_data['DAYS_BIRTH']
app_data['car_to_employ_ratio'] = app_data['OWN_CAR_AGE'] / app_data['DAYS_EMPLOYED']
app_data['children_ratio'] = app_data['CNT_CHILDREN'] / app_data['CNT_FAM_MEMBERS']
app_data['credit_to_annuity_ratio'] = app_data['AMT_CREDIT'] / app_data['AMT_ANNUITY']
app_data['credit_to_goods_ratio'] = app_data['AMT_CREDIT'] / app_data['AMT_GOODS_PRICE']
app_data['credit_to_income_ratio'] = app_data['AMT_CREDIT'] / app_data['AMT_INCOME_TOTAL']
app_data['days_employed_percentage'] = app_data['DAYS_EMPLOYED'] / app_data['DAYS_BIRTH']
app_data['income_credit_percentage'] = app_data['AMT_INCOME_TOTAL'] / app_data['AMT_CREDIT']
app_data['income_per_child'] = app_data['AMT_INCOME_TOTAL'] / (1 + app_data['CNT_CHILDREN'])
app_data['income_per_person'] = app_data['AMT_INCOME_TOTAL'] / app_data['CNT_FAM_MEMBERS']
app_data['payment_rate'] = app_data['AMT_ANNUITY'] / app_data['AMT_CREDIT']
app_data['phone_to_birth_ratio'] = app_data['DAYS_LAST_PHONE_CHANGE'] / app_data['DAYS_BIRTH']
app_data['phone_to_employ_ratio'] = app_data['DAYS_LAST_PHONE_CHANGE'] / app_data['DAYS_EMPLOYED']

app_data['cnt_non_child'] = app_data['CNT_FAM_MEMBERS'] - app_data['CNT_CHILDREN']
app_data['child_to_non_child_ratio'] = app_data['CNT_CHILDREN'] / app_data['cnt_non_child']
app_data['income_per_non_child'] = app_data['AMT_INCOME_TOTAL'] / app_data['cnt_non_child']
app_data['credit_per_person'] = app_data['AMT_CREDIT'] / app_data['CNT_FAM_MEMBERS']
app_data['credit_per_child'] = app_data['AMT_CREDIT'] / (1 + app_data['CNT_CHILDREN'])
app_data['credit_per_non_child'] = app_data['AMT_CREDIT'] / app_data['cnt_non_child']

app_data['short_employment'] = (app_data['DAYS_EMPLOYED'] < -2000).astype(int)
app_data['young_age'] = (app_data['DAYS_BIRTH'] < -14000).astype(int)
engg_features = ['annuity_income_percentage',
                'car_to_birth_ratio',
                'car_to_employ_ratio',
                'children_ratio',
                'credit_to_annuity_ratio',
                'credit_to_goods_ratio',
                'credit_to_income_ratio',
                'days_employed_percentage',
                'income_credit_percentage',
                'income_per_child',
                'income_per_person',
                'payment_rate',
                'phone_to_birth_ratio',
                'phone_to_employ_ratio',
                'cnt_non_child',
                'child_to_non_child_ratio',
                'income_per_non_child',
                'credit_per_person',
                'credit_per_child',
                'credit_per_non_child',
                'short_employment',
                'young_age']<jupyter_output><empty_output><jupyter_text>GroupBy features  Aggregation Recipies<jupyter_code>AGGREGATION_RECIPIES = [
    (['CODE_GENDER', 'NAME_EDUCATION_TYPE'], [('AMT_ANNUITY', 'max'),
                                              ('AMT_CREDIT', 'max'),
                                              ('EXT_SOURCE_1', 'mean'),
                                              ('EXT_SOURCE_2', 'mean'),
                                              ('OWN_CAR_AGE', 'max'),
                                              ('OWN_CAR_AGE', 'sum')]),
    (['CODE_GENDER', 'ORGANIZATION_TYPE'], [('AMT_ANNUITY', 'mean'),
                                            ('AMT_INCOME_TOTAL', 'mean'),
                                            ('DAYS_REGISTRATION', 'mean'),
                                            ('EXT_SOURCE_1', 'mean')]),
    (['CODE_GENDER', 'REG_CITY_NOT_WORK_CITY'], [('AMT_ANNUITY', 'mean'),
                                                 ('CNT_CHILDREN', 'mean'),
                                                 ('DAYS_ID_PUBLISH', 'mean')]),
    (['CODE_GENDER', 'NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE', 'REG_CITY_NOT_WORK_CITY'], [('EXT_SOURCE_1', 'mean'),
                                                                                           ('EXT_SOURCE_2', 'mean')]),
    (['NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE'], [('AMT_CREDIT', 'mean'),
                                                  ('AMT_REQ_CREDIT_BUREAU_YEAR', 'mean'),
                                                  ('APARTMENTS_AVG', 'mean'),
                                                  ('BASEMENTAREA_AVG', 'mean'),
                                                  ('EXT_SOURCE_1', 'mean'),
                                                  ('EXT_SOURCE_2', 'mean'),
                                                  ('EXT_SOURCE_3', 'mean'),
                                                  ('NONLIVINGAREA_AVG', 'mean'),
                                                  ('OWN_CAR_AGE', 'mean'),
                                                  ('YEARS_BUILD_AVG', 'mean')]),
    (['NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE', 'REG_CITY_NOT_WORK_CITY'], [('ELEVATORS_AVG', 'mean'),
                                                                            ('EXT_SOURCE_1', 'mean')]),
    (['OCCUPATION_TYPE'], [('AMT_ANNUITY', 'mean'),
                           ('CNT_CHILDREN', 'mean'),
                           ('CNT_FAM_MEMBERS', 'mean'),
                           ('DAYS_BIRTH', 'mean'),
                           ('DAYS_EMPLOYED', 'mean'),
                           ('DAYS_ID_PUBLISH', 'mean'),
                           ('DAYS_REGISTRATION', 'mean'),
                           ('EXT_SOURCE_1', 'mean'),
                           ('EXT_SOURCE_2', 'mean'),
                           ('EXT_SOURCE_3', 'mean')]),
]
groupby_aggregate_names = []
for groupby_cols, specs in tqdm(AGGREGATION_RECIPIES):
    group_object = app_data.groupby(groupby_cols)
    for select, agg in tqdm(specs):
        groupby_aggregate_name = '{}_{}_{}'.format('_'.join(groupby_cols), agg, select)
        app_data = app_data.merge(group_object[select]
                    .agg(agg)
                    .reset_index()
                    .rename(index=str,
                            columns={select: groupby_aggregate_name})
                    [groupby_cols + [groupby_aggregate_name]],
                    on=groupby_cols,
                    how='left')
        groupby_aggregate_names.append(groupby_aggregate_name)
diff_feature_names = []
for groupby_cols, specs in AGGREGATION_RECIPIES:
    for select, agg in tqdm(specs):
        if agg in ['mean', 'median', 'max', 'min']:
            groupby_aggregate_name = '{}_{}_{}'.format('_'.join(groupby_cols), agg, select)
            diff_name = '{}_diff'.format(groupby_aggregate_name)
            abs_diff_name = '{}_abs_diff'.format(groupby_aggregate_name)

            app_data[diff_name] = app_data[select] - app_data[groupby_aggregate_name]
            app_data[abs_diff_name] = np.abs(app_data[select] - app_data[groupby_aggregate_name])

            diff_feature_names.append(diff_name)
            diff_feature_names.append(abs_diff_name)
app_persist_path = persist_dir + '/application_train.pkl'
joblib.dump(app_data, app_persist_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1352886775003303, 'fraction_numerical': 0.003963535473642489, 'mean_word_length': 2.215802888700085, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 45, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '288282', 'license': 'MIT', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2387, 'n_words': 268, 'path': 'notebooks/application_feats.ipynb', 'repo_name': 'remidi/home-credit-default', 'size': '7569'}