<jupyter_start><jupyter_text>Solution 5 Hand crafted features<jupyter_code>def _status_to_int(status):
    if status in ['X', 'C']:
        return 0
    if pd.isnull(status):
        return np.nan
    return int(status)
bureau_balance['bureau_balance_dpd_level'] = bureau_balance['STATUS'].apply(_status_to_int)
bureau_balance['bureau_balance_status_unknown'] = (bureau_balance['STATUS'] == 'X').astype(int)
bureau_balance.head()
groupby = bureau_balance.groupby(['SK_ID_CURR'])
features = pd.DataFrame({'SK_ID_CURR': bureau_balance['SK_ID_CURR'].unique()})
def last_k_installment_features(gr, periods):
    gr_ = gr.copy()

    features = {}
    for period in periods:
        if period > 10e10:
            period_name = 'all_installment_'
            gr_period = gr_.copy()
        else:
            period_name = 'last_{}_'.format(period)
            gr_period = gr_[gr_['MONTHS_BALANCE'] >= (-1) * period]

        features = add_features_in_group(features, gr_period, 'bureau_balance_dpd_level',
                                             ['sum', 'mean', 'max', 'std', 'skew', 'kurt'],
                                             period_name)
        features = add_features_in_group(features, gr_period, 'bureau_balance_status_unknown',
                                             ['sum', 'mean'],
                                             period_name)
    return features
func = partial(last_k_installment_features, periods=[6, 12, 24, 60, 10e16])
g = parallel_apply(groupby, func, index_name='SK_ID_CURR', num_workers=10, chunk_size=10000).reset_index()
features = features.merge(g, on='SK_ID_CURR', how='left')<jupyter_output><empty_output><jupyter_text>Trend features<jupyter_code>def trend_in_last_k_installment_features(gr, periods):
    gr_ = gr.copy()
    gr_.sort_values(['MONTHS_BALANCE'], ascending=False, inplace=True)

    features = {}
    for period in periods:
        gr_period = gr_[gr_['MONTHS_BALANCE'] >= (-1) * period]

        features = add_trend_feature(features, gr_period,
                                         'bureau_balance_dpd_level', '{}_period_trend_'.format(period)
                                         )
    return features

def add_trend_feature(features, gr, feature_name, prefix):
    y = gr[feature_name].values
    try:
        x = np.arange(0, len(y)).reshape(-1, 1)
        lr = LinearRegression()
        lr.fit(x, y)
        trend = lr.coef_[0]
    except:
        trend = np.nan
    features['{}{}'.format(prefix, feature_name)] = trend
    return features
func = partial(trend_in_last_k_installment_features, periods=[6,12,24,60])
g = parallel_apply(groupby, func, index_name='SK_ID_CURR', num_workers=10, chunk_size=10000).reset_index()
features = features.merge(g, on='SK_ID_CURR', how='left')
features.head()
def last_k_instalment_fractions(old_features, fraction_periods):
    features = old_features[['SK_ID_CURR']].copy()
    
    for short_period, long_period in fraction_periods:
        short_feature_names = _get_feature_names(old_features, short_period)
        long_feature_names = _get_feature_names(old_features, long_period)
        
        for short_feature, long_feature in zip(short_feature_names, long_feature_names):
            old_name_chunk = '_{}_'.format(short_period)
            new_name_chunk ='_{}by{}_fraction_'.format(short_period, long_period)
            fraction_feature_name = short_feature.replace(old_name_chunk, new_name_chunk)
            features[fraction_feature_name] = old_features[short_feature]/old_features[long_feature]
    return pd.DataFrame(features).fillna(0.0)

def _get_feature_names(features, period):
    return sorted([feat for feat in features.keys() if '_{}_'.format(period) in feat])
g = last_k_instalment_fractions(features, fraction_periods=[(6, 12), (6, 24), (12,24), (12, 60)])
features = features.merge(g, on='SK_ID_CURR', how='left')

display(features.head())
X = application.merge(features, on='SK_ID_CURR',how='left')[features.columns.tolist() + ['TARGET']]
X_corr = abs(X.corr())
X_corr.sort_values('TARGET', ascending=False)['TARGET']
regex = '_history'
X_corr_truncated = X_corr.sort_values('TARGET', ascending=False).filter(regex=regex, axis=0)
X_corr_truncated['TARGET']
colnames = X_corr_truncated.index.tolist() + ['TARGET']
sns.heatmap(X_corr_truncated[colnames], 
            xticklabels=colnames,
            yticklabels=colnames)
X_to_plot = X[X.columns[0:20].tolist()+['TARGET']]
X_corr_to_plot = abs(X_to_plot.corr())
X_corr_to_plot.sort_values('TARGET', ascending=False)['TARGET']
sns.heatmap(X_corr_to_plot, 
            xticklabels=X_corr_to_plot.columns,
            yticklabels=X_corr_to_plot.columns)
import matplotlib.pyplot as plt
columns_to_use = X.columns[20:40]

for col in columns_to_use:
    print(col)
    X[col].hist(bins=50).plot()
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12933884297520662, 'fraction_numerical': 0.014462809917355372, 'mean_word_length': 3.6954413191076623, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '317885', 'license': 'MIT', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1590, 'n_words': 259, 'path': 'notebooks/eda-bureau_balance.ipynb', 'repo_name': 'JerryCatLeung/open-solution-home-credit', 'size': '4840'}