<jupyter_start><jupyter_text>Imports<jupyter_code>!pip install sklearn --upgrade
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import os 

from sklearn.linear_model import LinearRegression,LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.svm import SVC

from xgboost import XGBClassifier<jupyter_output>C:\Users\Bruguet\anaconda3\lib\site-packages\xgboost\compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.
  from pandas import MultiIndex, Int64Index<jupyter_text>Loading data<jupyter_code>employee= pd.read_csv('C:/py/data/attrition/employee_process.csv')
y=employee['y']
X=employee.drop(columns=['y'])<jupyter_output><empty_output><jupyter_text>Split into train and test subsample<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
X_train = pd.DataFrame(X_train, columns=X.columns)
X_test = pd.DataFrame(X_test, columns=X.columns)<jupyter_output><empty_output><jupyter_text>Defition of the metrics function<jupyter_code>from sklearn.metrics import confusion_matrix
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import f1_score,accuracy_score
from sklearn.metrics import precision_recall_curve, PrecisionRecallDisplay
def recall_precision(y_test,y_pred,model):
    precision, recall, thresholds = precision_recall_curve(y_test,y_pred)
    # convert to f score
    fscore = (2 * precision * recall) / (precision + recall)
    fscore[np.isnan(fscore)] = 0
    #print(fscore)
    # locate the index of the largest f score
    ix = np.argmax(fscore)
    optimal_threshold = thresholds[ix]

    # plot the recall precision curve for the model
    plt.plot(recall, precision, marker='.', label=model)
    no_skill = len(y_test[y_test==1]) / len(y_test)
    plt.plot([0,1], [no_skill,no_skill], linestyle='--', label='No Skill')
    plt.scatter(recall[ix], precision[ix], marker='o', color='black', label='Best')
    # axis labels
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.legend()
    # show the plot
    plt.show()
    
    print("\nTEST")
    print('Best Threshold=%f, F-Score=%.3f' % (optimal_threshold, fscore[ix]))
    y_pred2= np.where(y_pred <= optimal_threshold , 0, 1)
    cm = confusion_matrix(y_test, y_pred2)
    print("Confusion matrix \n" , cm)
    precision = cm[1,1]/sum(cm[:,1])
    print("\nPrecision : " + str(precision))
    recall = cm[1,1]/sum(cm[1,:])
    print("Recall : " + str(recall))
    f1 = 2*((precision*recall)/(precision+recall))
    print("F1 : " + str(f1))
    
    residual = (y_test - y_pred)
    print("\n Residuals distribution : \n", residual.describe())
    
    return optimal_threshold<jupyter_output><empty_output><jupyter_text>OLS<jupyter_code>model = LinearRegression().fit(X_train, y_train)
y_ols = model.predict(X_test)
t1 = recall_precision(y_test,y_ols,'Ordinary least square')
y_olsf1= np.where(y_ols <= t1 , 0, 1)<jupyter_output><empty_output><jupyter_text>Logistic regression<jupyter_code>clf = LogisticRegression(penalty='l2',C=0.1).fit(X_train, y_train)
y_log = clf.predict_proba(X_test)[:,1]
t2 = recall_precision(y_test,y_log,'Logistic regression')
y_logf1= np.where(y_log <= t2 , 0, 1)<jupyter_output><empty_output><jupyter_text>Support vector machine (SVM) Small dataset so C is put to the lowest even if not selected by cross-validation<jupyter_code>svc = SVC(kernel = 'sigmoid',
           C = 0.1,
           gamma = 0.01,
           random_state = 0, probability = True).fit(X_train, y_train)
y_svm = svc.predict_proba(X_test)[:,1]
'''svc = SVC(kernel = 'linear',
           C = 0.1,
           gamma = 'auto',
           class_weight={1: weight_ratio},
           random_state = 0, probability = True).fit(X_train, y_train)
y_svm = svc.predict_proba(X_test)[:,1]'''
t4 = recall_precision(y_test,y_svm,'Optimize support vector machine')
y_svmf1= np.where(y_svm <= t4 , 0, 1)<jupyter_output><empty_output><jupyter_text>XGBoost Since we have a small training set, the gamma parameter is increase to 2 to keep the model conservative<jupyter_code>m = XGBClassifier(
    n_estimator = 100,
    learning_rate = 0.5,
    max_depth=2,
    gamma=2,
    reg_alpha=0.7, #Regularization in the similarity score to build the trees (L1)
    reg_lambda=0.7,  #Regularization in the similarity score to build the trees (L2)
    min_child_weight = 3
)

m.fit(X_train,y_train)
y_xboost = m.predict_proba(X_test)[:,1]
t5 = recall_precision(y_test,y_xboost,'XGBoost')
y_xboostf1 = np.where(y_xboost <= t5 , 0, 1)<jupyter_output><empty_output><jupyter_text>Model comparaisons<jupyter_code>sns.set(style="ticks", color_codes=True)
sns.distplot((y_test-y_ols), hist=False, kde=True, kde_kws = {'shade': False, 'linewidth' : 1},
             bins=50, color = 'blue',
             hist_kws={'edgecolor':'black'})

sns.distplot((y_test-y_svm), hist=False, kde=True, kde_kws = {'shade': False, 'linewidth' : 1},
             bins=50, color = 'orange',
             hist_kws={'edgecolor':'black'})

sns.distplot((y_test-y_xboost), hist=False, kde=True, kde_kws = {'shade': False, 'linewidth' : 1},
             bins=50, color = 'green',
             hist_kws={'edgecolor':'black'})

plt.xlabel('Residuals')
plt.ylabel('Density')
plt.legend(labels=["OLS","SVM","XGBoost"])
plt.title('Residuals distribution')<jupyter_output>C:\Users\Bruguet\anaconda3\lib\site-packages\seaborn\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).
  warnings.warn(msg, FutureWarning)
C:\Users\Bruguet\anaconda3\lib\site-packages\seaborn\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).
  warnings.warn(msg, FutureWarning)
C:\Users\Bruguet\anaconda3\lib\site-packages\seaborn\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or[...]<jupyter_text>Using PrecisionRecallDisplay.**from_estimators**<jupyter_code>def recall (y_test,y_pred,model) :
    
    precision, recall, thresholds = precision_recall_curve(y_test,y_pred)
    # convert to f score
    fscore = (2 * precision * recall) / (precision + recall)
    # locate the index of the largest f score
    ix = np.argmax(fscore)
    # plot the roc curve for the model
    recall_plot = plt.plot(recall, precision, marker='.', label=model)
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.legend()
    plt.title('Metric visualisation - Precision-Recall curve')
    # show the plot /bbox_to_anchor=(0.5, -0.4), loc='center', ncol=1
    return(recall_plot)
recall_ols = recall(y_test,y_ols, 'OLS')
recall_svm = recall(y_test,y_svm, 'SVM')
recall_xgboost = recall(y_test,y_xboost, 'XGBoost')
 f_score = pd.DataFrame({
    'Algorithm' : ['OLS','SVM','XGBoost'],
    'y_pred' : [y_olsf1,y_svmf1,y_xboostf1]
})
f1_list = []
for index, row in f_score.iterrows():
    f1 = f1_score(y_test, row['y_pred'], average='binary')
    f1_list.append(f1)
acc_list = []
for index, row in f_score.iterrows():
    acc = accuracy_score(y_test, row['y_pred'])
    acc_list.append(acc)
    
f_score['f1']=f1_list
f_score['acc']=acc_list
f_score = f_score.drop(['y_pred'], axis=1)
f_score.sort_values(by=['f1'],ascending=False)<jupyter_output><empty_output><jupyter_text>Save the best classifier<jupyter_code>! pip install pickle
import pickle
# save the classifier
filename = 'attrition_svm.pkl'
pickle.dump(svc, open(filename, 'wb'))
# load the model from disk
loaded_model = pickle.load(open(filename, 'rb'))
result = loaded_model.score(X_test, y_test)
print('The prediction SVM score is around', round(result,2) * 100 ,'%')<jupyter_output>The prediction SVM score is around 87.0 %
{'fraction_non_alphanumeric': 0.12976022566995768, 'fraction_numerical': 0.015749882463563705, 'mean_word_length': 5.125989920806336, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '130507', 'license': 'MIT', 'n_tokens_mistral': 3158, 'n_tokens_neox': 3028, 'n_words': 662, 'path': 'notebooks/ml-classifiers.ipynb', 'repo_name': 'bgtm/attrition_prediction', 'size': '8508'}