<jupyter_start><jupyter_text>Preprocessing Solution 3  Feature Engineering Solution 3  Aggregations<jupyter_code>PREVIOUS_APPLICATION_AGGREGATION_RECIPIES = []
for agg in ['mean', 'min', 'max', 'sum', 'var']:
    for select in ['AMT_ANNUITY',
                   'AMT_APPLICATION',
                   'AMT_CREDIT',
                   'AMT_DOWN_PAYMENT',
                   'AMT_GOODS_PRICE',
                   'CNT_PAYMENT',
                   'DAYS_DECISION',
                   'HOUR_APPR_PROCESS_START',
                   'RATE_DOWN_PAYMENT'
                   ]:
        PREVIOUS_APPLICATION_AGGREGATION_RECIPIES.append((select, agg))
PREVIOUS_APPLICATION_AGGREGATION_RECIPIES = [(['SK_ID_CURR'], PREVIOUS_APPLICATION_AGGREGATION_RECIPIES)]
groupby_aggregate_names = []
for groupby_cols, specs in tqdm(PREVIOUS_APPLICATION_AGGREGATION_RECIPIES):
    group_object = previous_application.groupby(groupby_cols)
    for select, agg in tqdm(specs):
        groupby_aggregate_name = '{}_{}_{}'.format('_'.join(groupby_cols), agg, select)
        application = application.merge(group_object[select]
                              .agg(agg)
                              .reset_index()
                              .rename(index=str,
                                      columns={select: groupby_aggregate_name})
                              [groupby_cols + [groupby_aggregate_name]],
                              on=groupby_cols,
                              how='left')
        groupby_aggregate_names.append(groupby_aggregate_name)
application.head()
application_agg = application[groupby_aggregate_names + ['TARGET']]
application_agg_corr = abs(application_agg.corr())
application_agg_corr.sort_values('TARGET', ascending=False)['TARGET']<jupyter_output><empty_output><jupyter_text>Solution 4 Hand crafted features<jupyter_code>numbers_of_applications = [1, 3, 5]
features = pd.DataFrame({'SK_ID_CURR': previous_application['SK_ID_CURR'].unique()})
prev_applications_sorted = previous_application.sort_values(['SK_ID_CURR', 'DAYS_DECISION'])
group_object = prev_applications_sorted.groupby(by=['SK_ID_CURR'])['SK_ID_PREV'].nunique().reset_index()
group_object.rename(index=str,
                    columns={'SK_ID_PREV': 'previous_application_number_of_prev_application'},
                    inplace=True)
features = features.merge(group_object, on=['SK_ID_CURR'], how='left')
prev_applications_sorted['previous_application_prev_was_approved'] = (
        prev_applications_sorted['NAME_CONTRACT_STATUS'] == 'Approved').astype('int')
group_object = prev_applications_sorted.groupby(by=['SK_ID_CURR'])[
    'previous_application_prev_was_approved'].last().reset_index()
features = features.merge(group_object, on=['SK_ID_CURR'], how='left')
prev_applications_sorted['previous_application_prev_was_refused'] = (
        prev_applications_sorted['NAME_CONTRACT_STATUS'] == 'Refused').astype('int')
group_object = prev_applications_sorted.groupby(by=['SK_ID_CURR'])[
    'previous_application_prev_was_refused'].last().reset_index()
features = features.merge(group_object, on=['SK_ID_CURR'], how='left')
for number in numbers_of_applications:
    prev_applications_tail = prev_applications_sorted.groupby(by=['SK_ID_CURR']).tail(number)

    group_object = prev_applications_tail.groupby(by=['SK_ID_CURR'])['CNT_PAYMENT'].mean().reset_index()
    group_object.rename(index=str, columns={
        'CNT_PAYMENT': 'previous_application_term_of_last_{}_credits_mean'.format(number)},
                        inplace=True)
    features = features.merge(group_object, on=['SK_ID_CURR'], how='left')

    group_object = prev_applications_tail.groupby(by=['SK_ID_CURR'])['DAYS_DECISION'].mean().reset_index()
    group_object.rename(index=str, columns={
        'DAYS_DECISION': 'previous_application_days_decision_about_last_{}_credits_mean'.format(number)},
                        inplace=True)
    features = features.merge(group_object, on=['SK_ID_CURR'], how='left')

    group_object = prev_applications_tail.groupby(by=['SK_ID_CURR'])['DAYS_FIRST_DRAWING'].mean().reset_index()
    group_object.rename(index=str, columns={
        'DAYS_FIRST_DRAWING': 'previous_application_days_first_drawing_last_{}_credits_mean'.format(number)},
                        inplace=True)
    features = features.merge(group_object, on=['SK_ID_CURR'], how='left')
features.head()
X = application.merge(features,
                                left_on=['SK_ID_CURR'],
                                right_on=['SK_ID_CURR'],
                                how='left',
                                validate='one_to_one')
engineered_numerical_columns = list(features.columns)
engineered_numerical_columns.remove('SK_ID_CURR')
credit_eng = X[engineered_numerical_columns + ['TARGET']]
credit_eng_corr = abs(credit_eng.corr())
credit_eng_corr.sort_values('TARGET', ascending=False)['TARGET']
sns.heatmap(credit_eng_corr, 
            xticklabels=credit_eng_corr.columns,
            yticklabels=credit_eng_corr.columns)<jupyter_output><empty_output><jupyter_text>Solution 6 Hand crafted features<jupyter_code>features = pd.DataFrame({'SK_ID_CURR': previous_application['SK_ID_CURR'].unique()})
prev_app_sorted = previous_application.sort_values(['SK_ID_CURR', 'DAYS_DECISION'])
prev_app_sorted['approved'] = (prev_app_sorted['NAME_CONTRACT_STATUS'] == 'Approved').astype('int')
prev_app_sorted['refused'] = (prev_app_sorted['NAME_CONTRACT_STATUS'] == 'Refused').astype('int')
prev_app_sorted['revolving_loan'] = (prev_app_sorted['NAME_CONTRACT_TYPE'] == 'Revolving loans').astype('int')
prev_app_sorted['cash_loan'] = (prev_app_sorted['NAME_CONTRACT_TYPE'] == 'Cash loans').astype('int')
prev_app_sorted['consumer_loan'] = (prev_app_sorted['NAME_CONTRACT_TYPE'] == 'Consumer loans').astype('int')
prev_app_sorted['credit_goods_diff'] = prev_app_sorted['AMT_CREDIT'] - prev_app_sorted['AMT_GOODS_PRICE']
prev_app_sorted['credit_goods_ratio'] = prev_app_sorted['AMT_CREDIT'] / prev_app_sorted['AMT_GOODS_PRICE']
prev_app_sorted['application_credit_diff'] = prev_app_sorted['AMT_APPLICATION'] - prev_app_sorted['AMT_CREDIT']
prev_app_sorted['application_credit_ratio'] = prev_app_sorted['AMT_APPLICATION'] / prev_app_sorted['AMT_CREDIT']
prev_app_sorted['NAME_PRODUCT_TYPE_x_sell'] = (prev_app_sorted['NAME_PRODUCT_TYPE'] == 'x-sell').astype('int')
prev_app_sorted['NAME_PRODUCT_TYPE_walk_in'] = (prev_app_sorted['NAME_PRODUCT_TYPE'] == 'walk-in').astype('int')
prev_app_sorted['NAME_PAYMENT_TYPE_bank'] = (prev_app_sorted['NAME_PAYMENT_TYPE'] == 'Cash through the bank').astype('int')
prev_app_sorted['NAME_PAYMENT_TYPE_account'] = (prev_app_sorted['NAME_PAYMENT_TYPE'] == 'Non-cash from your account').astype('int')
prev_app_sorted['NAME_PAYMENT_TYPE_employer'] = (prev_app_sorted['NAME_PAYMENT_TYPE'] == 'Cashless from the account of the employer').astype('int')
prev_app_sorted_groupby = prev_app_sorted.groupby(by=['SK_ID_CURR'])
g = prev_app_sorted_groupby['SK_ID_PREV'].agg('nunique').reset_index()
g.rename(index=str, columns={'SK_ID_PREV': 'previous_application_number_of_prev_application'}, inplace=True)
features = features.merge(g, on=['SK_ID_CURR'], how='left')
g = prev_app_sorted_groupby['refused'].mean().reset_index()
g.rename(index=str, columns={'refused': 'previous_application_fraction_of_refused_applications'}, inplace=True)
features = features.merge(g, on=['SK_ID_CURR'], how='left')
features.head()
def _get_last_k_applications_feature_name(feature_name, number, suffix):
    return 'previous_application_{}_last_{}_applications_{}'.format(feature_name, number, suffix)


def get_last_k_credits_features(prev_app_sorted, numbers_of_applications):
    features = pd.DataFrame({'SK_ID_CURR': prev_app_sorted['SK_ID_CURR'].unique()})

    feature_list = ['CNT_PAYMENT', 'DAYS_DECISION', 'DAYS_FIRST_DRAWING', 'NFLAG_INSURED_ON_APPROVAL',
                        'NAME_PRODUCT_TYPE_x_sell', 'NAME_PRODUCT_TYPE_walk_in', 'NAME_PAYMENT_TYPE_bank',
                        'NAME_PAYMENT_TYPE_account', 'NAME_PAYMENT_TYPE_employer', 'approved', 'refused',
                        'revolving_loan', 'cash_loan', 'consumer_loan', 'credit_goods_diff', 'credit_goods_ratio',
                        'application_credit_diff', 'application_credit_ratio']

    for number in numbers_of_applications:
        prev_applications_tail = prev_app_sorted.groupby('SK_ID_CURR').tail(number)
        tail_groupby = prev_applications_tail.groupby('SK_ID_CURR')
        g = tail_groupby[feature_list].agg('mean')

        g = g.rename(axis='columns', mapper=partial(_get_last_k_applications_feature_name,
                                                        number=number, suffix='mean')).reset_index()
        features = features.merge(g, how='left', on='SK_ID_CURR')

    return features
from functools import partial

g = get_last_k_credits_features(prev_app_sorted, numbers_of_applications=numbers_of_applications)
features = features.merge(g, on=['SK_ID_CURR'], how='left')
features.head()
X = application.merge(features,
                                left_on=['SK_ID_CURR'],
                                right_on=['SK_ID_CURR'],
                                how='left',
                                validate='one_to_one')
engineered_numerical_columns = list(features.columns)
engineered_numerical_columns.remove('SK_ID_CURR')
credit_eng = X[engineered_numerical_columns + ['TARGET']]
credit_eng_corr = abs(credit_eng.corr())
credit_eng_corr.sort_values('TARGET', ascending=False)['TARGET']
plt.figure(figsize=(15, 15))
sns.heatmap(credit_eng_corr, 
            xticklabels=credit_eng_corr.columns,
            yticklabels=credit_eng_corr.columns)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12833230006199628, 'fraction_numerical': 0.0011365984707584211, 'mean_word_length': 4.660233918128655, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '340800', 'license': 'MIT', 'n_tokens_mistral': 3347, 'n_tokens_neox': 3186, 'n_words': 348, 'path': 'notebooks/eda-previous_application.ipynb', 'repo_name': 'robbiefrost/open-solution-home-credit', 'size': '9678'}