<jupyter_start><jupyter_text>1. Logistic Regression<jupyter_code>lm = LogisticRegression(solver='newton-cg',  # For comparison, use the same solver as statsmodels default
                          C=100000)  # No regularization

lm.fit(X_train, y_train)
# print('intercept: ', round(lm.intercept_[0], 4))
# pd.DataFrame(list(zip(cat_variables,np.transpose(lm.coef_))),columns=['Feature','Coef'])\
# .sort_values('Coef',ascending=False)
# Coefficient for top features

feature = X_train.columns
coef = pd.DataFrame(list(zip((feature),lm.coef_[0])),columns = ['Feature','Coefficient']).\
sort_values(['Coefficient'],ascending=False)
top_feature = coef.head(10)
top_feature
list(top_feature.Feature)
# plt.scatter(x='Feature',y='Coefficient',data=top_feature)
bottom_feature = coef.tail(10)
bottom_feature
y_predict = lm.predict(X_test)
pd.DataFrame(confusion_matrix(y_test, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
print('Accuracy score:',accuracy_score(y_test,y_predict),'\n')
print('Classification report (Logistic regression):\n\n',classification_report(y_test,y_predict))
# Logistic regression ROC curve
y_scores = lm.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'b', label = 'AUC_lr = %0.3f' % roc_auc)

plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.title('ROC Curve of Logistic regression')
plt.show()
# Logistic regression ROC curve (different color)
y_scores = lm.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)


fig, ax = plt.subplots()

plt.plot(fpr, tpr, 'cadetblue', linewidth=3, label = 'AUC = %0.3f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'k--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate',fontsize=14)
plt.xlabel('False Positive Rate',fontsize=14)
plt.title('ROC Curve of Logistic regression',fontsize=16)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12);

# fig.savefig('../plot/ROC_lr_1.pdf', bbox_inches='tight', transparent=True)
roc_df = pd.DataFrame(roc_curve(y_test, y_scores[:, 1])).transpose()
roc_df.columns=['fpr','tpr','threshold']
# roc_df.to_csv('../dump/roc_df_lr.csv')
roc_df
sns.set()
fig, ax = plt.subplots()

# pd.DataFrame(tpr,fpr).plot.area(alpha=0.4)
y_scores = lm.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)

plt.stackplot(fpr, tpr,alpha=0.3)
plt.plot(fpr, tpr, 'cadetblue', linewidth=3.5, label = 'Logistic Regression = %0.3f' % roc_auc)
# plt.plot([0, 1], [0, 1],'k--',label = 'Random guess = 0.5')

plt.legend(loc = 'lower right',bbox_to_anchor=(1.6, 0))
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate',fontsize=14)
plt.xlabel('False Positive Rate',fontsize=14)
plt.title('Model Performance (ROC)',fontsize=16)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12);

# fig.savefig('../plot/ROC_lr_2.pdf', bbox_inches='tight', transparent=True)
# Create a concise df of test data candidate info, target, and probability
df = df_test
df = df.reset_index()
prob_df = pd.DataFrame(lm.predict_proba(X_test)).apply(lambda x:round(x,3)*100)

df = pd.concat([df,prob_df],axis=1)[['enrollee_id','experience',1,'target']]
df.columns=['Enrollee_id','experience','Probability (%)','Target']
df = df.sort_values('Probability (%)',ascending=False).reset_index()

# Save as csv file
# df.to_csv('../dump/candidate_proba.csv')
df<jupyter_output><empty_output><jupyter_text>Grid search cross-validationLogistic regression does not really have hyperparameters to tune.* **solver** : ‘newton-cg’, ‘lbfgs’, ‘liblinear’, ‘sag’, ‘saga’* **penalty** (regularization) : ‘none’, ‘l1’, ‘l2’, ‘elasticnet’* **C** (penalty strength) : 100, 10, 1.0, 0.1, 0.01<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RepeatedStratifiedKFold
lm_2=LogisticRegression()
solvers = ['newton-cg', 'lbfgs', 'liblinear']
penalty = ['none', 'l1', 'l2', 'elasticnet']
c_values = [100000000, 10000, 1000, 100, 1.0, 0.1, 0.01]
# c_values = [1.5, 1.2, 0.8, 0.4, 0.2]
# Define grid search
grid = dict(solver=solvers,penalty=penalty,C=c_values)
# cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
grid_search = GridSearchCV(estimator=lm_2, param_grid=grid, n_jobs=-1, cv=5,\
                           scoring='precision',error_score=0)
grid_result = grid_search.fit(X_train, y_train)
# summarize results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
# for mean, stdev, param in zip(means, stds, params):
#     print("%f (%f) with: %r" % (mean, stdev, param))

lm_2 = grid_search.best_estimator_
# LogisticRegression(solver='lbfgs', C=10000, penalty='l2')
lm_2.fit(X_train,y_train)
y_predict = lm_2.predict(X_test)
pd.DataFrame(confusion_matrix(y_test, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
print('Accuracy score:',accuracy_score(y_test,y_predict))
print('ROC AUC score:',roc_auc_score(y_test,lm_2.predict_proba(X_test)[:, 1]),'\n')
print('Classification report (Logistic regression):\n\n',classification_report(y_test,y_predict))
# Precision and recall for (1) both improved 0.01...<jupyter_output><empty_output><jupyter_text>Cross-validation<jupyter_code>from sklearn.model_selection import cross_val_score
recalls = cross_val_score(lm_2, X_train, y_train, cv=5, scoring='recall')
print('Mean:', np.mean(recalls))
print('All recalls:', (recalls))<jupyter_output>Mean: 0.67616741611899
All recalls: [0.76271186 0.72154964 0.62850225 0.62193013 0.6461432 ]<jupyter_text>* Adjust threshold<jupyter_code>def make_confusion_matrix(model, threshold=0.5):
    # Predict class 1 if probability of being in class 1 is greater than threshold
    # (model.predict(X_test) does this automatically with a threshold of 0.5)
    y_predict = (model.predict_proba(X_test)[:, 1] >= threshold)
    fraud_confusion = confusion_matrix(y_test, y_predict)
    plt.figure(dpi=80)
    sns.heatmap(fraud_confusion, cmap=plt.cm.Blues, annot=True, square=True, fmt='d',
           xticklabels=['Not Looking', 'Looking'],
           yticklabels=['Not Looking', 'Looking']);
    plt.xlabel('prediction')
    plt.ylabel('actual')
# Let's see how our confusion matrix changes with changes to the cutoff! 

from ipywidgets import interactive, FloatSlider
interactive(lambda threshold: make_confusion_matrix(lm_2, threshold), threshold=(0.0,1.0,0.02))
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
len(threshold)
threshold[0],threshold[685]
value=0.2
def find_nearest(array, value):
        array = np.asarray(array)
        idx = (np.abs(array - value)).argmin()
        return idx
    
i = find_nearest(threshold, value)
def roc_threshold(lm,threshold):
    y_scores = lm.predict_proba(X_test)
    fpr, tpr, threshold_array = roc_curve(y_test, y_scores[:, 1])
    roc_auc = auc(fpr, tpr)

    def find_nearest(array, value):
        array = np.asarray(array)
        idx = (np.abs(array - value)).argmin()
        return idx
    
    i = find_nearest(threshold_array, threshold)
    
    fig, ax = plt.subplots()
    

    # pd.DataFrame(tpr,fpr).plot.area(alpha=0.4)
    plt.stackplot(fpr, tpr,alpha=0.3)
    plt.plot(fpr, tpr, 'cadetblue', linewidth=3.5, label = 'Logistic Regression')
    plt.plot([0, 1], [0, 1],'k--',linewidth=1, label = 'Random Guess')
    plt.scatter(fpr[i], tpr[i],linewidth=3,color='orange',zorder=10)
    ax.annotate('Threshold= %.2f' % (threshold_array[i]),[fpr[i]+0.03,tpr[i]-0.02])

    plt.legend(loc = 'lower right')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('True Positive Rate',fontsize=14)
    plt.xlabel('False Positive Rate',fontsize=14)
    plt.title('Model Performance (ROC)',fontsize=16)
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12);
from ipywidgets import interactive, FloatSlider
interactive(lambda threshold: roc_threshold(lm_2, threshold), threshold=(0,1,0.2))
threshold = 0.4
y_predict = (lm_2.predict_proba(X_test)[:, 1] >= threshold)
print('Classification report (Logistic regression):\n\n',classification_report(y_test,y_predict))
# LogisticRegression(solver='lbfgs', C=10000, penalty='l2')
lm_2.fit(X_train,y_train)
y_predict = (lm_2.predict_proba(X_test)[:, 1] >= threshold)
predict_df = pd.DataFrame(confusion_matrix(y_test, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
predict_df = predict_df.append(predict_df.sum(numeric_only=True), ignore_index=True)
predict_df.index=['Not changing (0)','Changing (1)','Total']
predict_df
# Classification report on **train data**
threshold = 0.3
y_predict = (lm_2.predict_proba(X_train)[:, 1] >= threshold)
print('Classification report (Logistic regression):\n\n',classification_report(y_train,y_predict))
# LogisticRegression(solver='lbfgs', C=10000, penalty='l2')
lm_2.fit(X_train,y_train)
y_predict = lm_2.predict(X_train)
predict_df = pd.DataFrame(confusion_matrix(y_train, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
predict_df = predict_df.append(predict_df.sum(numeric_only=True), ignore_index=True)
predict_df.index=['Not changing (0)','Changing (1)','Total']
predict_df
# Use recall to find threshold
threshold_list = [x/100 for x in range(0,100)]
# print(threshold_list)
recall_list=[]
for value in threshold_list:
    threshold = value
    y_predict = (lm_2.predict_proba(X_test)[:, 1] >= threshold)
    recall_list.append(recall_score(y_test,y_predict))

i = find_nearest(recall_list, 0.8)
# Compared to baseline Logistic regression model with 0.41 precision, recall has improved from 0.61 to 0.82!<jupyter_output><empty_output><jupyter_text>* Feature Engineering<jupyter_code>X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=2)
X_train['training_hours_experience'] = X_train['training_hours'] * X_train['experience']
X_test['training_hours_experience'] = X_test['training_hours'] * X_test['experience']

X_train['last_job_experience'] = X_train['experience'] / X_train['last_new_job']
X_test['last_job_experience'] = X_test['experience'] / X_test['last_new_job']

X_train['last_job_city'] = X_train['last_new_job'] * X_train['city_development_index']
X_test['last_job_city'] = X_test['last_new_job'] * X_test['city_development_index']

X_train['training_hours_city'] = X_train['training_hours'] * X_train['city_development_index']
X_test['training_hours_city'] = X_test['training_hours'] * X_test['city_development_index']
scaler = StandardScaler()
X_train['last_job_experience'] = scaler.fit_transform(X_train[['last_job_experience']])
X_train
lm = LogisticRegression(solver='newton-cg',  # For comparison, use the same solver as statsmodels default
                          C=100000)  # No regularization

lm.fit(X_train, y_train)
y_predict = lm.predict(X_test)
pd.DataFrame(confusion_matrix(y_test, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
lm = LogisticRegression(solver='newton-cg',  # For comparison, use the same solver as statsmodels default
                          C=100000)  # No regularization

lm.fit(X_train, y_train)
y_predict = lm.predict(X_test)
pd.DataFrame(confusion_matrix(y_test, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
lm = LogisticRegression(solver='newton-cg',  # For comparison, use the same solver as statsmodels default
                          C=100000)  # No regularization

lm.fit(X_train, y_train)
y_predict = lm.predict(X_test)
pd.DataFrame(confusion_matrix(y_test, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
lm = LogisticRegression(solver='newton-cg',  # For comparison, use the same solver as statsmodels default
                          C=100000)  # No regularization

lm.fit(X_train, y_train)
y_predict = lm.predict(X_test)
pd.DataFrame(confusion_matrix(y_test, y_predict),\
             columns=['Predict-not changing (0)','Predict-change (1)'],\
            index=['Not changing (0)','Changing (1)'])
# Interaction terms do not improve the model<jupyter_output><empty_output><jupyter_text>2. KNN<jupyter_code># Reload data (after train test split for feature engineering)
# All variables (continuous + categorical)
X_train = pd.read_pickle('../dump/X_adasyn')
y_train = pd.read_pickle('../dump/y_adasyn')
X_test = pd.read_pickle('../dump/X_test_processed')
y_test = df_test['target']
from sklearn.neighbors import KNeighborsClassifier

knn_3 = KNeighborsClassifier(n_neighbors=3)
knn_3.fit(X_train,y_train)
pred = knn_3.predict(X_test)

print('Accuracy score:',accuracy_score(y_test,pred))
print('ROC AUC score:',roc_auc_score(y_test,knn_3.predict_proba(X_test)[:, 1]),'\n')
print('Classification report (kNN, n=3):\n\n',classification_report(y_test,pred))
knn_5 = KNeighborsClassifier(n_neighbors=5)
knn_5.fit(X_train,y_train)
pred = knn_5.predict(X_test)

print('Accuracy score:',accuracy_score(y_test,pred))
print('ROC AUC score:',roc_auc_score(y_test,knn_5.predict_proba(X_test)[:, 1]),'\n')
print('Classification report (kNN, n=5):\n\n',classification_report(y_test,pred))
# knn (n=3)
y_scores = knn_3.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'b', label = 'AUC_knn3 = %0.3f' % roc_auc)

# knn (n=5)
y_scores = knn_5.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'g', label = 'AUC_knn5 = %0.3f' % roc_auc)

plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.title('ROC Curve of kNN')
plt.show()<jupyter_output><empty_output><jupyter_text>3. Naive Bayes<jupyter_code>nb = GaussianNB()
nb.fit(X_train,y_train)
pred = nb.predict(X_test)
print('Accuracy score:',accuracy_score(y_test,pred))
print('ROC AUC score:',roc_auc_score(y_test,nb.predict_proba(X_test)[:, 1]),'\n')
print('Classification report (Naive Bayes):\n\n',classification_report(y_test,pred))
# Naive Bayes ROC curve
y_scores = nb.predict_proba(X_test)
fpr, tpr, threshold = roc_curve(y_test, y_scores[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, 'b', label = 'AUC_nb = %0.3f' % roc_auc)

plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.title('ROC Curve of Naive Bayes')
plt.show()
# Compared to baseline version, Naive Bayes has higher recall but lower precision.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1641976927110645, 'fraction_numerical': 0.029431043523859467, 'mean_word_length': 5.832512315270936, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '471342', 'license': 'MIT', 'n_tokens_mistral': 6148, 'n_tokens_neox': 5749, 'n_words': 1060, 'path': 'notebook/08_refine_models_1.ipynb', 'repo_name': 'katiehuang1221/onl_ds5_project_3', 'size': '15256'}