<jupyter_start><jupyter_text>** Two stage problem. *** Default Classifier: Use AUC or F1-score to evaluate performance because of class imbalance.* Default Loss: Use MAE ( Evaluation metric ) to evaluate performance<jupyter_code>%matplotlib inline

import pandas as pd
import numpy as np
import os, sys

from sklearn.cross_validation import train_test_split, StratifiedKFold
from sklearn.metrics import mean_absolute_error
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import MinMaxScaler, Imputer
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier, GradientBoostingClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import f_regression, SelectKBest
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score, f1_score

import xgboost as xgb

import matplotlib.pyplot as plt
import seaborn as sns

import warnings
warnings.filterwarnings('ignore')

np.random.seed(3)

basepath = os.path.expanduser('~/Desktop/src/Loan_Default_Prediction/')
sys.path.append(os.path.join(basepath, 'src'))

from data import *
from features import *
from models import *
train  = pd.read_csv(os.path.join(basepath, 'data/raw/train_v2.csv'),
                     index_col='id',
                     dtype=np.float32
                    )
# test   = pd.read_csv(os.path.join(basepath, 'data/raw/test_v2.csv'), index_col='id')
# sample_sub = pd.read_csv(os.path.join(basepath, 'data/raw/sampleSubmission.csv'))
# loss target
train['is_default'] = (train.loss > 0).astype(np.int)<jupyter_output><empty_output><jupyter_text>** Training examples are in the chronological order but not in the test set. **<jupyter_code>train = train.iloc[np.random.permutation(len(train))] # shuffle training set.
# features to remove from training set
features_to_remove = ['f33', 'f678', 'f37', 'f764', 'f700', \
                      'f34', 'f38', 'f702', 'f701', 'f736', 'f35']

features_to_remove.extend(['is_default', 'loss'])
itrain, itest = get_stratified_sample(train, train.is_default, train_size=.3, random_state=11)

X_train = train.iloc[itrain][train.columns.drop(features_to_remove)]
X_test  = train.iloc[itest][train.columns.drop(features_to_remove)]

y_train = train.is_default.iloc[itrain]
y_test  = train.is_default.iloc[itest] 
print(X_train.shape, X_test.shape)
del train<jupyter_output><empty_output><jupyter_text>** Two Step Modelling. ** * Pipeline for predicting whether there was any loss.* Pipeline to predict the actual value in case any loss was incurred.<jupyter_code>pipeline_default = Pipeline([
        ('feature_union', FeatureUnion([
                    ('golden_feature', GoldenFeatures())
                ])),
        ('imputer', Imputer()),
        ('scaler', MinMaxScaler()),
        ('select', TreeBasedSelection(ExtraTreesClassifier(), y_train, n_features_to_select=30)),
        ('union', FeatureUnion([
                    ('feature_interaction', FeatureInteraction())
                ])),
        ('model', RandomForestClassifier(n_estimators=25, n_jobs=2, random_state=5))
    ])
pipeline_default.fit(X_train, y_train)
yhat  = pipeline_default.predict_proba(X_test)[:, 1]
print('AUC score: %f'%(roc_auc_score(y_test, yhat)))
best_f1_score = bestF1(y_test, yhat)
pipeline_loss = Pipeline([
        ('feature_union', FeatureUnion([
                    ('golden_feature', GoldenFeatures())
                ])),
        ('imputer', Imputer()),
        ('scaler', MinMaxScaler()),
        ('select', TreeBasedSelection(ExtraTreesClassifier(), y_train, n_features_to_select=30)),
        ('union', FeatureUnion([
                    ('feature_interaction', FeatureInteraction())
                ])),
        ('model', RandomForestClassifier(n_estimators=25, n_jobs=2, random_state=5))
    ])<jupyter_output><empty_output><jupyter_text>** Submission **<jupyter_code>sample_sub = pd.read_csv(os.path.join(basepath, 'data/raw/sampleSubmission.csv'))
sample_sub['loss'] = loss
sample_sub.to_csv(os.path.join(basepath, 'submissions/baseline.csv'), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12496960855823, 'fraction_numerical': 0.01312910284463895, 'mean_word_length': 4.522147651006711, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '488794', 'license': 'MIT', 'n_tokens_mistral': 1365, 'n_tokens_neox': 1320, 'n_words': 271, 'path': 'notebooks/BenchmarkModel.ipynb', 'repo_name': 'numb3r33/Loan-Default-Prediction', 'size': '4113'}