<jupyter_start><jupyter_text>Import libraries<jupyter_code>import pandas as pd 
import numpy as np 
import scipy as sp 
import seaborn as sns 
import matplotlib.pyplot as plt 
import datetime as dt 

from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.ensemble import RandomForestClassifier 
from sklearn.ensemble.partial_dependence import partial_dependence, plot_partial_dependence

from xgboost import XGBClassifier 
from xgboost import cv

from collections import Counter
import sys, os, glob

print(dt.datetime.now(), 'python version:')
print(sys.version)
print(dt.datetime.now(), 'pandas version:', pd.__version__)<jupyter_output>2019-03-14 13:18:17.861596 python version:
3.6.7 (default, Dec  5 2018, 15:02:05) 
[GCC 4.8.5 20150623 (Red Hat 4.8.5-36)]
2019-03-14 13:18:17.862421 pandas version: 0.24.1<jupyter_text>Read data<jupyter_code>print(dt.datetime.now(), 'start reading data...')

if os.path.isfile('../input/train.csv'): 
    print('remote environment')
    dirc = '../input/'
    train = pd.read_csv(dirc + 'train.csv')
    test = pd.read_csv(dirc + 'test.csv')
    submission = pd.read_csv(dirc + 'sample_submission.csv')
else: 
    print('local environment')
    dirc = '../../../data/Santander-Customer-Transaction-Prediction/'
    train = pd.read_csv(dirc + 'train.csv.zip', compression='zip')
    test = pd.read_csv(dirc + 'test.csv.zip', compression='zip')
    submission = pd.read_csv(dirc + 'sample_submission.csv.zip', compression='zip')


x = train.drop(['ID_code', 'target'], axis=1)
y = train.target

print('train shape', train.shape)
print('test shape', test.shape)

print(dt.datetime.now(), 'finish reading data')

test.drop('ID_code', axis=1, inplace=True)<jupyter_output>2019-03-14 14:49:29.231547 start reading data...
local environment
train shape (200000, 202)
test shape (200000, 201)
2019-03-14 14:49:50.469663 finish reading data<jupyter_text>Cross validation test<jupyter_code>n_sp = 5 

sfk = StratifiedKFold(n_splits=n_sp)
pred_train_mtx = np.zeros(len(train))
pred_test_mtx = np.zeros((len(test), n_sp))

i = 0
for train_idx, val_idx in sfk.split(x, y): 
    train_x = x.iloc[train_idx, :] 
    val_x   = x.iloc[val_idx, :]
    train_y = y.iloc[train_idx] 
    val_y   = y.iloc[val_idx]
    
    print(dt.datetime.now(), 'cycle', i, 'start training...')  
    
    model = XGBClassifier(n_estimators=100000, n_jobs=32, tree_method='gpu_hist', 
                          learning_rate=0.01, 
                          max_delta_step=2, colsample_bylevel=0.6, colsample_bytree=0.1, 
                          gamma=0.004, max_bin=256, max_depth=8, max_leaves=27, min_child_weight=96, 
                          reg_alpha=0.003, reg_lambda=0.060, subsample=0.4) 

    #model = XGBClassifier(tree_method='gpu_hist')
    model.fit(train_x, train_y, eval_set=[(val_x, val_y)], 
              verbose=500,
              eval_metric='auc', early_stopping_rounds=200)#2500)
    
    pred_temp = model.predict_proba(val_x)[:, 1]
    print('roc_auc_score:', roc_auc_score(val_y, pred_temp))
    
    pred_train_mtx[val_idx] = pred_temp 
    pred_test_mtx[:, i] = model.predict_proba(test)[:, 1]
    
    print(dt.datetime.now(), 'finish cycle', i, 'training') 
    print()
    
    i += 1

print('final auc score:', roc_auc_score(y, pred_train_mtx))
pred = pred_test_mtx.mean(axis = 1)<jupyter_output>2019-03-14 13:49:44.372957 cycle 0 start training...
[0]	validation_0-auc:0.620153
Will train until validation_0-auc hasn't improved in 200 rounds.
[500]	validation_0-auc:0.866979<jupyter_text>Eval function versus training steps<jupyter_code>eval_auc = model.evals_result()['validation_0']['auc']

plt.figure(figsize=(10, 6.18))
plt.xlabel('training step')
plt.ylabel('auc')
sns.lineplot(x = range(len(eval_auc)), y = eval_auc)<jupyter_output><empty_output><jupyter_text>Receiver operating characteristic curve<jupyter_code>fpr, tpr, thresholds = roc_curve(y, pred_train_mtx)
score = roc_auc_score(y, pred_train_mtx)

plt.figure()
plt.xlabel('False Positive Rate (FPR)')
plt.ylabel('True Positive Rate (TPR)')
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.plot(fpr, fpr, linestyle='--')
plt.plot(fpr, tpr, label='AUC_ROC: '+str(score))
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Submit<jupyter_code>submission['target'] = pred
submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14109188946304202, 'fraction_numerical': 0.05459447315210065, 'mean_word_length': 4.516728624535316, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '613614', 'license': 'MIT', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1665, 'n_words': 314, 'path': 'competitions/Santander-Customer-Transaction-Prediction/kernel.ipynb', 'repo_name': 'society765/kaggle_project', 'size': '4451'}