<jupyter_start><jupyter_text>Feature Selection for D-Features  import libraries<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from lightgbm import LGBMClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import RFECV<jupyter_output><empty_output><jupyter_text>change working directory<jupyter_code>os.chdir('..')<jupyter_output><empty_output><jupyter_text>load data<jupyter_code>dataframe = pd.read_csv('data/all_D_features.csv')
dataframe.head()<jupyter_output><empty_output><jupyter_text>get some basic info about the data<jupyter_code>dataframe.describe()<jupyter_output><empty_output><jupyter_text>remove useless column (row ID?)<jupyter_code>dataframe = dataframe.drop('Unnamed: 0', axis=1)
dataframe.head()<jupyter_output><empty_output><jupyter_text>define utility function to reduce memory usage<jupyter_code>def reduce_mem_usage(df, verbose=True):
    """
    Reduce dataframe size

    params:
    - df: dataframe to reduce the size of

    return:
    - dataframe of reduced size
    """
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64', 'float128']
    start_mem = df.memory_usage().sum() / 1024**2    

    for col in df.columns:
        col_type = df[col].dtypes

        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()

            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                elif c_min > np.finfo(np.float64).min and c_max < np.finfo(np.float64).max:
                    df[col] = df[col].astype(np.float64)
                elif c_min > np.finfo(np.float128).min and c_max < np.finfo(np.float128).max:
                    df[col] = df[col].astype(np.float128)
                    
    end_mem = df.memory_usage().sum() / 1024**2

    if verbose: 
        print(
            'Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(
            end_mem, 100 * (start_mem - end_mem) / start_mem
        ))

    return df<jupyter_output><empty_output><jupyter_text>reduce dataframe size to reduce memory usage<jupyter_code>dataframe = reduce_mem_usage(dataframe)<jupyter_output>Mem. usage decreased to 17.46 Mb (75.8% reduction)<jupyter_text>check percentage NaNs in each column<jupyter_code>for column in dataframe.columns:
    num_nans = dataframe[column].isna().sum()
    print('percentage of NaNs for {}: {:.3f}'.format(
        column, 
        num_nans / len(dataframe) * 100
    ))<jupyter_output>percentage of NaNs for D1: 0.215
percentage of NaNs for D2: 47.549
percentage of NaNs for D3: 44.515
percentage of NaNs for D4: 28.605
percentage of NaNs for D5: 52.467
percentage of NaNs for D6: 87.607
percentage of NaNs for D7: 93.410
percentage of NaNs for D8: 87.312
percentage of NaNs for D9: 87.312
percentage of NaNs for D10: 12.873
percentage of NaNs for D11: 47.293
percentage of NaNs for D12: 89.041
percentage of NaNs for D13: 89.509
percentage of NaNs for D14: 89.469
percentage of NaNs for D15: 15.090
percentage of NaNs for isFraud: 0.000<jupyter_text>remove columns with a lot of NaNs- if we do imputation when there is a lot of NaNs, the imputed value may not be representative of actual value<jupyter_code># keep columns with less than 60% NaNs
dataframe = dataframe.loc[:, dataframe.isna().mean() < 0.6]
dataframe.head()<jupyter_output><empty_output><jupyter_text>define function for drawing correlation heatmap<jupyter_code>def draw_correlation_heatmap(df):
    """
    params:
    - df: dataframe of which correlation heatmap is drawn
    """
    fig, ax = plt.subplots(figsize=(10,10))

    sns.heatmap(
        df.corr(), 
        vmax=1.0, 
        center=0, 
        fmt='.2f',
        square=True, 
        linewidths=0.5, 
        annot=True, 
        cbar_kws={'shrink': .70},
    )

    plt.show()<jupyter_output><empty_output><jupyter_text>draw correlation heatmap to check for correlated features- correlated features may affect classifier performance<jupyter_code>draw_correlation_heatmap(dataframe)<jupyter_output><empty_output><jupyter_text>drop highly-correlated features<jupyter_code>dataframe = dataframe.drop(['D2'], axis=1)  # D1 and D2 has correlation coefficient of 0.98
dataframe.head()<jupyter_output><empty_output><jupyter_text>do imputation to fill in NaNs<jupyter_code># replace NaN with interquartile mean
dataframe = dataframe.fillna(dataframe.quantile().mean())
dataframe.head()<jupyter_output><empty_output><jupyter_text>separate data into features and target variable<jupyter_code>d_features_dataframe = dataframe.drop('isFraud', axis=1)
is_fraud_data = dataframe['isFraud']<jupyter_output><empty_output><jupyter_text>define function for doing RFECV- Recursive Feature Elimination with Cross Validation<jupyter_code>def do_rfecv(features_dataframe, target_variable, classifier, verbose=True):
    """
    Do Recursive Feature Elimination with Cross Validation.
    Scoring method: ROC AUC

    params:
    - features_dataframe: dataframe that contain features only
    - target_variable: series that contain the target variable
    - classifier: classifier for evaluating feature importance
    - verbose (bool): if True, prints info

    return:
    - rfecv
    """
    rfecv = RFECV(classifier, scoring='roc_auc', n_jobs=2)
    rfecv = rfecv.fit(features_dataframe, target_variable)

    if verbose:
        print(f'Optimal number of features: {rfecv.n_features_}')
        print(f'ROC AUC: {rfecv.grid_scores_[rfecv.n_features_ - 1]}')

    return rfecv<jupyter_output><empty_output><jupyter_text>do Recursive Feature Elimination with Cross Validation (RFECV)- to select most important features- use random forest for evaluation<jupyter_code>%%time

rf_classifier = RandomForestClassifier(n_jobs=2)
rfecv = do_rfecv(d_features_dataframe, is_fraud_data, rf_classifier)<jupyter_output>Optimal number of features: 7
ROC AUC: 0.7106897506899459
CPU times: user 2min 14s, sys: 219 ms, total: 2min 14s
Wall time: 9min 58s<jupyter_text>define function to plot accuracy vs number of features<jupyter_code>def plot_acc_vs_num_features(grid_scores, method_used):
    plt.figure(figsize=(16, 9))
    plt.title(method_used, fontsize=18, fontweight='bold', pad=20)

    plt.xlabel('Number of features selected', fontsize=14, labelpad=20)
    plt.ylabel('% Correct Classification', fontsize=14, labelpad=20)

    plt.plot(
        range(1, len(grid_scores) + 1), 
        grid_scores, 
        color='#303F9F', 
        linewidth=3,
    )

    plt.show()<jupyter_output><empty_output><jupyter_text>plot line chart of accuracy vs number of features<jupyter_code>plot_acc_vs_num_features(
    rfecv.grid_scores_, 
    'Recursive Feature Elimination with Cross-Validation (using Random Forest)', 
)<jupyter_output><empty_output><jupyter_text>show feature ranking<jupyter_code>for column, rank in zip(d_features_dataframe.columns, rfecv.ranking_):
    print(f'feature: {column} rank: {rank}')<jupyter_output>feature: D1 rank: 1
feature: D3 rank: 1
feature: D4 rank: 1
feature: D5 rank: 1
feature: D10 rank: 1
feature: D11 rank: 1
feature: D15 rank: 1<jupyter_text>define function to get only the most important features<jupyter_code>def get_important_features(features_dataframe, rfecv, verbose=True):
    least_important_features = np.where(rfecv.support_ == False)[0]

    if verbose:
        print('not-important features:', list(features_dataframe.columns[least_important_features]))

    important_features_dataframe = features_dataframe.drop(
        features_dataframe.columns[least_important_features], 
        axis=1
    )

    if verbose:
        print('important features:', list(important_features_dataframe.columns))

    return important_features_dataframe<jupyter_output><empty_output><jupyter_text>keep only the most important features<jupyter_code>most_important_features = get_important_features(d_features_dataframe, rfecv)<jupyter_output>not-important features: []
important features: ['D1', 'D3', 'D4', 'D5', 'D10', 'D11', 'D15']<jupyter_text>define function to plot feature importance bar chart<jupyter_code>def plot_feature_importance_chart(important_features_dataframe, rfecv):
    plot_dataframe = pd.DataFrame()

    plot_dataframe['attr'] = important_features_dataframe.columns
    plot_dataframe['importance'] = rfecv.estimator_.feature_importances_

    plot_dataframe = plot_dataframe.sort_values(by='importance', ascending=False)

    plt.figure(figsize=(16, 14))
    plt.barh(
        y=plot_dataframe['attr'], 
        width=plot_dataframe['importance'], 
        color='#1976D2',
    )

    plt.title('RFECV - Feature Importances', fontsize=20, fontweight='bold', pad=20)
    plt.xlabel('Importance', fontsize=14, labelpad=20)

    plt.show()<jupyter_output><empty_output><jupyter_text>plot bar chart for feature importance<jupyter_code>plot_feature_importance_chart(most_important_features, rfecv)<jupyter_output><empty_output><jupyter_text>do RFECV again- to select most important features- use light gradient boosting machine for evaluation<jupyter_code>%%time

lgbm_classifier = LGBMClassifier()
rfecv = do_rfecv(d_features_dataframe, is_fraud_data, lgbm_classifier)<jupyter_output>Optimal number of features: 7
ROC AUC: 0.7679062923604875
CPU times: user 38.4 s, sys: 480 ms, total: 38.9 s
Wall time: 41.6 s<jupyter_text>plot line chart of accuracy vs number of features<jupyter_code>plot_acc_vs_num_features(
    rfecv.grid_scores_, 
    'Recursive Feature Elimination with Cross-Validation (using LightGBM)', 
)<jupyter_output><empty_output><jupyter_text>show feature ranking<jupyter_code>for column, rank in zip(d_features_dataframe.columns, rfecv.ranking_):
    print(f'feature: {column} rank: {rank}')<jupyter_output>feature: D1 rank: 1
feature: D3 rank: 1
feature: D4 rank: 1
feature: D5 rank: 1
feature: D10 rank: 1
feature: D11 rank: 1
feature: D15 rank: 1<jupyter_text>keep only the most important features<jupyter_code>most_important_features = get_important_features(d_features_dataframe, rfecv)<jupyter_output>not-important features: []
important features: ['D1', 'D3', 'D4', 'D5', 'D10', 'D11', 'D15']<jupyter_text>plot bar chart for feature importance<jupyter_code>plot_feature_importance_chart(most_important_features, rfecv)<jupyter_output><empty_output><jupyter_text>apparently, [decision trees can deal with multicollinearity](https://towardsdatascience.com/why-feature-correlation-matters-a-lot-847e8ba439c4)- check if using D2 would increase ROC AUC<jupyter_code># not dropping D2 this time
dataframe = pd.read_csv('data/all_D_features.csv')
dataframe = dataframe.drop('Unnamed: 0', axis=1)
dataframe = reduce_mem_usage(dataframe)
dataframe = dataframe.loc[:, dataframe.isna().mean() < 0.6]
dataframe = dataframe.fillna(dataframe.quantile().mean())<jupyter_output>Mem. usage decreased to 17.46 Mb (75.8% reduction)<jupyter_text>separate data into features and target variable<jupyter_code>d_features_dataframe = dataframe.drop('isFraud', axis=1)
is_fraud_data = dataframe['isFraud']<jupyter_output><empty_output><jupyter_text>do RFECV- to select most important features- use random forest for evaluation<jupyter_code>%%time

rf_classifier = RandomForestClassifier(n_jobs=2)
rfecv = do_rfecv(d_features_dataframe, is_fraud_data, rf_classifier)<jupyter_output>Optimal number of features: 8
ROC AUC: 0.7197007048704398
CPU times: user 2min 21s, sys: 216 ms, total: 2min 21s
Wall time: 11min 33s<jupyter_text>plot line chart of accuracy vs number of features<jupyter_code>plot_acc_vs_num_features(
    rfecv.grid_scores_, 
    'Recursive Feature Elimination with Cross-Validation (using Random Forest)', 
)<jupyter_output><empty_output><jupyter_text>show feature ranking<jupyter_code>for column, rank in zip(d_features_dataframe.columns, rfecv.ranking_):
    print(f'feature: {column} rank: {rank}')<jupyter_output>feature: D1 rank: 1
feature: D2 rank: 1
feature: D3 rank: 1
feature: D4 rank: 1
feature: D5 rank: 1
feature: D10 rank: 1
feature: D11 rank: 1
feature: D15 rank: 1<jupyter_text>keep only the most important features<jupyter_code>most_important_features = get_important_features(d_features_dataframe, rfecv)<jupyter_output>not-important features: []
important features: ['D1', 'D2', 'D3', 'D4', 'D5', 'D10', 'D11', 'D15']<jupyter_text>plot bar chart for feature importance<jupyter_code>plot_feature_importance_chart(most_important_features, rfecv)<jupyter_output><empty_output><jupyter_text>do RFECV again- to select most important features- use light gradient boosting machine for evaluation<jupyter_code>%%time

lgbm_classifier = LGBMClassifier()
rfecv = do_rfecv(d_features_dataframe, is_fraud_data, lgbm_classifier)<jupyter_output>Optimal number of features: 8
ROC AUC: 0.77288522852928
CPU times: user 39.7 s, sys: 392 ms, total: 40.1 s
Wall time: 47.2 s<jupyter_text>plot line chart of accuracy vs number of features<jupyter_code>plot_acc_vs_num_features(
    rfecv.grid_scores_, 
    'Recursive Feature Elimination with Cross-Validation (using LightGBM)', 
)<jupyter_output><empty_output><jupyter_text>show feature ranking<jupyter_code>for column, rank in zip(d_features_dataframe.columns, rfecv.ranking_):
    print(f'feature: {column} rank: {rank}')<jupyter_output>feature: D1 rank: 1
feature: D2 rank: 1
feature: D3 rank: 1
feature: D4 rank: 1
feature: D5 rank: 1
feature: D10 rank: 1
feature: D11 rank: 1
feature: D15 rank: 1<jupyter_text>keep only the most important features<jupyter_code>most_important_features = get_important_features(d_features_dataframe, rfecv)<jupyter_output>not-important features: []
important features: ['D1', 'D2', 'D3', 'D4', 'D5', 'D10', 'D11', 'D15']<jupyter_text>plot bar chart for feature importance<jupyter_code>plot_feature_importance_chart(most_important_features, rfecv)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10655457590048527, 'fraction_numerical': 0.035267582530244, 'mean_word_length': 4.964940888707705, 'pattern_counts': {'":': 0, '<': 156, '<?xml version=': 0, '>': 154, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '40', 'dir': 'jupyter-structured-clean-dedup', 'id': '626509', 'license': 'MIT', 'n_tokens_mistral': 5507, 'n_tokens_neox': 5016, 'n_words': 1150, 'path': 'project/feature_selection/D_feature_selection.ipynb', 'repo_name': 'NgoJunHaoJason/NTU-CZ4041', 'size': '14631'}