<jupyter_start><jupyter_text>Global Volunteer Application Qualifier A Machine Learning model to identify application most likely to approve.> This notebook is an study and exercise on data science pipeline and modeling, by <NAME>1. Importing packages and dataset2. Selecting features and spliting dataset3. Training XGBoost Model4. K-fold Cross Validation (with and without time series)5. Feature importance6. ROC and P|R Curve--- 1. Importing packages and dataset<jupyter_code>import os
import gc
import re
import sys
import traceback
import warnings
import datetime
from time import time

import seaborn as sns
from matplotlib import pyplot as plt


import numpy as np
import pandas as pd
pd.options.display.max_columns = None
#import lightgbm as lgb
import xgboost as xgb
from sklearn import metrics
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
from category_encoders.target_encoder import TargetEncoder

from tqdm import tqdm_notebook
from IPython.display import display, HTML
import ds_utils

%load_ext autoreload
%autoreload 2

gc.enable()
warnings.filterwarnings("ignore")
np.random.seed(710105)
df = pd.read_csv("../Data/dataset_gv_v2.csv")

df.columns<jupyter_output><empty_output><jupyter_text>2. Selecting features and spliting dataset<jupyter_code>num_cols = [
 'required_skills_percentage',
 'required_languages_percentage',
 'required_backgrounds_percentage',
 'month_signup',
 'day_signup',
 'abs_date_signup',
 'month_applied',
 'day_applied',
 'abs_date_applied',
 'time_to_apply',
 'person_accepted_count',
 'person_rejected_count',
 'person_applications_count',
 'professional_experience_in_years',
 'english_level',
 'spanish_level',
]
cat_cols = [
 'host_lc',
 'host_mc',
# 'program',
 'home_lc',
# 'home_mc',
 'latest_academic_level.name'
]
bool_cols = [
 'profile_completeness_ge',
 'profile_completeness_gv',
 'profile_completeness_gt',
]
date_cols = [
 'date_signup',
 'date_applied',
]
target = "approval"
X = df[num_cols+bool_cols+cat_cols]
y = df[target]
#split sets

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30)
# Label Encoding
for f in X.columns:
    if X_train[f].dtype=='object' or X_test[f].dtype=='object':
        X_train[f] = X_train[f].fillna('NA')
        X_test[f] = X_test[f].fillna('NA')
        te = TargetEncoder(smoothing=10, min_samples_leaf=100)
        te.fit(X_train[f],y_train)
        X_train[f] = te.transform(X_train[f])
        X_test[f] = te.transform(X_test[f])<jupyter_output><empty_output><jupyter_text>3. Training XGBoost Model<jupyter_code>params = {
          'min_child_weight': 0.03454472573214212,
          'feature_fraction': 0.3797454081646243,
          'bagging_fraction': 0.4181193142567742,
          'min_data_in_leaf': 106,
          'objective': 'binary:logistic',
          'max_depth': 6,
          'learning_rate': 0.006883242363721497,
          "boosting_type": "gbdt",
          "bagging_seed": 11,
          "metric": 'auc',
          'reg_alpha': 0.3899927210061127,
          'reg_lambda': 0.6485237330340494,
          "nrounds":200
         }
#train model

clf = xgb.XGBClassifier(**params)
_ = clf.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>4. K-fold Cross Validation (with and without time series)<jupyter_code># k-fold cross-validation

from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score

kf = KFold(n_splits=10) 

cv_scores = cross_val_score(clf,X_train,y_train,scoring = "roc_auc",cv=kf)
print("perda: {0:.3f} (+/- {1:.3f})".format(cv_scores.mean(), cv_scores.std()))
kf = KFold(n_splits=5)

tprs,fprs,aucs = [],[],[]




for train_index, test_index in kf.split(X):
    X_train, X_test = X.iloc[train_index,:], X.iloc[test_index,:]
    y_train, y_test = y.iloc[train_index], y.iloc[test_index]
    # Label Encoding
    for f in X.columns:
        if X_train[f].dtype=='object' or X_test[f].dtype=='object':
            X_train[f] = X_train[f].fillna('NA')
            X_test[f] = X_test[f].fillna('NA')
            te = TargetEncoder(smoothing=10, min_samples_leaf=100)
            te.fit(X_train[f],y_train)
            X_train[f] = te.transform(X_train[f])
            X_test[f] = te.transform(X_test[f])
    clf = xgb.XGBClassifier(**params)
    _ = clf.fit(X_train, y_train)
    y_pred = clf.predict_proba(X_test)[:,1]
    fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred)
    auc = metrics.auc(fpr, tpr)
    tprs += [tpr]
    fprs += [fpr]
    aucs += [auc]
    
fig, (ax1, ax2) = plt.subplots(1,2)
fig.set_size_inches(14, 6)
#ROC 
#f = plt.figure(1)
ax1.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'grey')

for i in range(5):
    ax1.plot(fprs[i], tprs[i], color='blue',
             label=r'Fold 1 ROC (AUC = %0.4f)' % (aucs[i]),lw=2, alpha=1)

ax1.set_xlabel('Índice de Falsos Positivos')
ax1.set_ylabel('Índice de Verdadeiros Positivos')
ax1.set_title('Média da ROC')
ax1.legend(loc="lower right")
tscv = TimeSeriesSplit(n_splits=5)

tprs,fprs,aucs = [],[],[]




for train_index, test_index in tscv.split(X):
    X_train, X_test = X.iloc[train_index,:], X.iloc[test_index,:]
    y_train, y_test = y.iloc[train_index], y.iloc[test_index]
    # Label Encoding
    for f in X.columns:
        if X_train[f].dtype=='object' or X_test[f].dtype=='object':
            X_train[f] = X_train[f].fillna('NA')
            X_test[f] = X_test[f].fillna('NA')
            te = TargetEncoder(smoothing=10, min_samples_leaf=100)
            te.fit(X_train[f],y_train)
            X_train[f] = te.transform(X_train[f])
            X_test[f] = te.transform(X_test[f])
    clf = xgb.XGBClassifier(**params)
    _ = clf.fit(X_train, y_train)
    y_pred = clf.predict_proba(X_test)[:,1]
    fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred)
    auc = metrics.auc(fpr, tpr)
    tprs += [tpr]
    fprs += [fpr]
    aucs += [auc]
    
fig, (ax1, ax2) = plt.subplots(1,2)
fig.set_size_inches(14, 6)
#ROC 
#f = plt.figure(1)
ax1.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'grey')

for i in range(5):
    ax1.plot(fprs[i], tprs[i], color='blue',
             label=r'Fold 1 ROC (AUC = %0.4f)' % (aucs[i]),lw=2, alpha=1)

ax1.set_xlabel('Índice de Falsos Positivos')
ax1.set_ylabel('Índice de Verdadeiros Positivos')
ax1.set_title('Média da ROC')
ax1.legend(loc="lower right")
#timeseries kfold

from sklearn.model_selection import TimeSeriesSplit

tscv = TimeSeriesSplit(n_splits=10)
tscv_scores = cross_val_score(clf, X_train, y_train, cv=tscv, scoring='roc_auc')
print("Perda: {0:.3f} (+/- {1:.3f})".format(tscv_scores.mean(), tscv_scores.std()))<jupyter_output>Perda: 0.923 (+/- 0.041)<jupyter_text>5. Feature importance<jupyter_code>feature_imp = pd.DataFrame(sorted(zip(clf.feature_importances_,X.columns)), columns=['Value','Feature'])

plt.figure(figsize=(18, 10))
sns.barplot(x="Value", y="Feature", data=feature_imp.sort_values(by="Value", ascending=False)[:40])
plt.title('LightGBM Feature Importance - Top 40')
plt.tight_layout()
plt.show()
#plt.savefig('lgbm_importances.png')<jupyter_output><empty_output><jupyter_text>6. ROC and P|R Curve<jupyter_code>y_pred = clf.predict_proba(X_test)[:,1]
fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred)
auc = metrics.auc(fpr, tpr)
precision, recall, th = metrics.precision_recall_curve(y_test, y_pred)
fig, (ax1, ax2) = plt.subplots(1, 2)
fig.set_size_inches(14, 6)
#ROC 
#f = plt.figure(1)
ax1.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'grey')

ax1.plot(fpr, tpr, color='blue',
         label=r'Fold 1 ROC (AUC = %0.4f)' % (auc),lw=2, alpha=1)

ax1.set_xlabel('Índice de Falsos Positivos')
ax1.set_ylabel('Índice de Verdadeiros Positivos')
ax1.set_title('Média da ROC')
ax1.legend(loc="lower right")

# PR plt
#g = plt.figure(2)
ax2.plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')

ax2.plot(recall, precision, color='blue',
         label=r'Fold 1 P|R')
ax2.set_xlabel('Recall')
ax2.set_ylabel('Precision')
ax2.set_title('Curva P|R')
_ = ax2.legend(loc="lower left")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15039643211100098, 'fraction_numerical': 0.03369672943508424, 'mean_word_length': 4.571428571428571, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '450266', 'license': 'MIT', 'n_tokens_mistral': 3417, 'n_tokens_neox': 3219, 'n_words': 589, 'path': '.ipynb_checkpoints/Global_Volunteer_Application_Qualifier-checkpoint.ipynb', 'repo_name': 'Brites101/application_qualifier', 'size': '8083'}