<jupyter_start><jupyter_text>Feature selection, modeling, parameter tuning, performance evaluation   Prepare notebook Import standard libraries<jupyter_code>import numpy as np 
import pandas as pd
import joblib
import pickle 
import random
import seaborn as sns
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Import my source functions<jupyter_code>import sys
import os
sys.path.append('../src/features')
from build_features import get_dummies
sys.path.append('../src/models')
from predict_model import clint_scorer, trump_scorer, avg_scorer
#warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Import estimators from scikit-learn<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC<jupyter_output><empty_output><jupyter_text>Import modules for feature selection and performance evaluation<jupyter_code>from sklearn.feature_selection import RFECV
from imblearn.over_sampling import SMOTE
from sklearn.metrics import confusion_matrix, make_scorer, classification_report
from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV, cross_validate<jupyter_output>Using TensorFlow backend.<jupyter_text>Set random seed for repeatability<jupyter_code>seed = 20<jupyter_output><empty_output><jupyter_text>Load predictors (no dummy variables yet) and target variable<jupyter_code>with open('../data/processed/iss_feats.pickle', 'rb') as file:
    X_no = pickle.load(file)
with open('../data/processed/target.pickle', 'rb') as file:
    y = pickle.load(file)<jupyter_output><empty_output><jupyter_text>Split data into train and test sets<jupyter_code>X_train_rfe, X_test_rfe, y_train_rfe, y_test_rfe = train_test_split(
    X_no, y, test_size=0.2, random_state=20)<jupyter_output><empty_output><jupyter_text>Define performance metrics  Create a custom scorer to pass to sklearn cross validator to evaluate performance of models<jupyter_code>clinton_metric = make_scorer(clint_scorer)
trump_metric = make_scorer(trump_scorer)
comb_metric = make_scorer(avg_scorer)<jupyter_output><empty_output><jupyter_text>Define models  Set up basic model objects<jupyter_code>lr = LogisticRegression(
    C=2, penalty='l2', solver='lbfgs', multi_class='auto', max_iter=1000, random_state=20)
rf = RandomForestClassifier(n_estimators=100, random_state=20)
knn = KNeighborsClassifier()
svm = SVC(kernel='linear', random_state=20)<jupyter_output><empty_output><jupyter_text>Evaluate training performance of basic models Create a list of model names and separate list of models<jupyter_code>name_list = ['Logistic regression', 'Random forest',
             'K-nearest neighbors', 'Support vector machine']
model_list = [lr, rf, knn, svm]<jupyter_output><empty_output><jupyter_text>Run cross validation on all basic models and score using Clinton metric<jupyter_code>for idx, model in enumerate(model_list):
    clint_scores = cross_val_score(
        model, X_train_rfe, y_train_rfe, cv=5, scoring=clinton_metric, n_jobs=-1)
    name = name_list[idx]
    print(name, ":", clint_scores, 'Mean:', round(clint_scores.mean(), 5))<jupyter_output>Logistic regression : [90.4 90.2 90.2 88.3 89.5] Mean: 89.72
Random forest : [88.8 89.3 88.8 88.3 90. ] Mean: 89.04
K-nearest neighbors : [87.2 87.  88.1 84.9 87.4] Mean: 86.92
Support vector machine : [90.7 91.3 90.9 88.8 90.6] Mean: 90.46<jupyter_text>Run cross validation on all basic models and score using Trump metric<jupyter_code>for idx, model in enumerate(model_list):
    trump_scores = cross_val_score(
        model, X_train_rfe, y_train_rfe, cv=5, scoring=trump_metric, n_jobs=-1)
    name = name_list[idx]
    print(name, ':', trump_scores, 'Mean:', round(trump_scores.mean(), 5))<jupyter_output>Logistic regression : [87.3 88.7 83.4 86.9 88. ] Mean: 86.86
Random forest : [89.3 91.4 84.1 87.8 87.7] Mean: 88.06
K-nearest neighbors : [84.1 86.9 80.7 86.4 83.4] Mean: 84.3
Support vector machine : [86.2 88.2 82.6 86.2 87.5] Mean: 86.14<jupyter_text>Run cross validation on all basic models and score with combined metric<jupyter_code>for idx, model in enumerate(model_list):
    comb_scores = cross_val_score(
        model, X_train_rfe, y_train_rfe, cv=5, scoring=comb_metric, n_jobs=-1)
    name = name_list[idx]
    print(name, ':', comb_scores,'Mean:', comb_scores.mean())<jupyter_output>Logistic regression : [88.85 89.45 86.8  87.6  88.75] Mean: 88.29
Random forest : [89.05 90.35 86.45 88.05 88.85] Mean: 88.55
K-nearest neighbors : [85.65 86.95 84.4  85.65 85.4 ] Mean: 85.60999999999999
Support vector machine : [88.45 89.75 86.75 87.5  89.05] Mean: 88.3<jupyter_text>Feature selection Recursive feature elimination on top performing basic models using combined metric and print optimal number of features found using each model<jupyter_code>selector_lr = RFECV(lr, step=1, min_features_to_select=1,  # logistic regression rfe
                    scoring=comb_metric, cv=5)
selector_lr.fit(X_train_rfe, y_train_rfe)  #
selector_rf = RFECV(rf, step=1, min_features_to_select=1,  # random forest rfe
                    scoring=comb_metric, cv=5)
selector_rf.fit(X_train_rfe, y_train_rfe)
print('Optimal number of LR features: {}'.format(selector_lr.n_features_))
print('Optimal number of RF features: {}'.format(selector_rf.n_features_))<jupyter_output>Optimal number of LR features: 7
Optimal number of RF features: 22<jupyter_text>Print the list of selected features from the logistic regression basic model<jupyter_code>rank_list = selector_lr.ranking_
feature_ranks = []
nt_slct_rank_feat = []
slct_rank_feat = []
for idx, rank in enumerate(rank_list):
    feature_ranks.append([rank, X_train_rfe.columns[idx]])
for list in feature_ranks:
    if list[0] != 1:
        nt_slct_rank_feat.append(list)
    if list[0] == 1:
        slct_rank_feat.append(list)
print('Not selected:' '\n', nt_slct_rank_feat, '\n')
print('Selected:''\n', slct_rank_feat)<jupyter_output>Not selected:
 [[3, 'imiss_a_2016'], [9, 'imiss_b_2016'], [5, 'imiss_d_2016'], [6, 'imiss_e_2016'], [2, 'imiss_g_2016'], [12, 'imiss_h_2016'], [4, 'imiss_i_2016'], [17, 'imiss_j_2016'], [14, 'imiss_k_2016'], [16, 'imiss_m_2016'], [11, 'imiss_n_2016'], [10, 'imiss_o_2016'], [7, 'imiss_q_2016'], [15, 'imiss_r_2016'], [8, 'imiss_s_2016'], [13, 'imiss_t_2016']] 

Selected:
 [[1, 'imiss_c_2016'], [1, 'imiss_f_2016'], [1, 'imiss_l_2016'], [1, 'imiss_p_2016'], [1, 'imiss_u_2016'], [1, 'imiss_x_2016'], [1, 'imiss_y_2016']]<jupyter_text>Pickle selected features as selected_feat.pickle<jupyter_code>slctd_feat = []
for item in slct_rank_feat:
    slctd_feat.append(item[1])
slctd_feat
selected_feat = X_no[slctd_feat]
with open('../data/processed/selected_feat.pickle', 'wb') as file:
    pickle.dump(selected_feat, file)<jupyter_output><empty_output><jupyter_text>Visualize relationship between performance and number of features<jupyter_code>plt.figure(figsize=(16, 9))
line_plt = sns.lineplot(range(1, len(selector_lr.grid_scores_) + 1), selector_lr.grid_scores_, color='grey', linewidth=3)
line_plt.set_title('Recursive Feature Elimination - Logistic Regression', pad=20, fontsize=18)
line_plt.set_xlabel('Number of features selected', fontsize=14, labelpad=20)
line_plt.set_ylabel('Percent correct', fontsize=14, labelpad=20)
plt.show()<jupyter_output><empty_output><jupyter_text>Drop the features that were not selected from the predictors dataframe<jupyter_code>np.where(selector_lr.support_ == False)[0]
X_selected = X_no.drop(X_no.columns[np.where(selector_lr.support_ == False)[0]], axis=1)
print(len(X_selected.columns))
X_selected.head()<jupyter_output>7<jupyter_text>Feature engineering Use one hot encoding to transform the selected X features into dummy variables<jupyter_code>X = get_dummies(X_selected)
len(X.columns)
X.columns<jupyter_output><empty_output><jupyter_text>Using the selected features that are now one-hot encoded, create a new train and test set<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=20)<jupyter_output><empty_output><jupyter_text>Parameter tuning Define parameter grid for random forest and run grid search cross validation<jupyter_code>rf_2 = RandomForestClassifier(n_estimators=100, random_state=20)
param_grid = {'n_estimators': [100], 'max_depth': [
    1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 'oob_score': [True], 'max_features': [None, 'sqrt']}
grid = GridSearchCV(rf_2, param_grid, iid=True, cv=5, n_jobs=-1,
                    scoring=comb_metric, verbose=1, return_train_score=True)<jupyter_output><empty_output><jupyter_text>Fit to training data and print best parameters and best score<jupyter_code>grid.fit(X_train, y_train)
print(grid.best_params_)
print(round(grid.best_score_, 3))
lr_2 = LogisticRegression(C=2, penalty='l2', solver='lbfgs',
                          multi_class='auto', max_iter=1000, random_state=20)
lr_2.fit(X_train, y_train)
print(lr_2.coef_[0])
print(lr_2.classes_)
print(lr_2.intercept_)
predictions_lr_2 = lr_2.predict(X_train)
scores_lr_2 = cross_validate(lr_2, X_train, y_train, scoring=comb_metric, cv=5)
scores_lr_2['test_score']<jupyter_output><empty_output><jupyter_text>Use SMOTE to oversample minority class Print counts of each vote category before oversampling<jupyter_code>print('Before oversampling, counts of label 1: {}'.format(sum(y_train == 1))) 
print('Before oversampling, counts of label 2: {}'.format(sum(y_train == 2))) 
print('Before oversampling, counts of label 3: {}'.format(sum(y_train == 3)))<jupyter_output>Before oversampling, counts of label 1: 2810
Before oversampling, counts of label 2: 2795
Before oversampling, counts of label 3: 795<jupyter_text>Create SMOTE object and fit to train data<jupyter_code>sm = SMOTE(random_state = 20) 
X_train_sm, y_train_sm = sm.fit_sample(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Print counts of each vote category after oversampling<jupyter_code>print('After oversampling, the shape of train_X: {}'.format(X_train_sm.shape)) 
print('After oversampling, the shape of train_y: {} \n'.format(y_train_sm.shape)) 
print('After oversampling, counts of label 1: {}'.format(sum(y_train_sm == 1))) 
print('After oversampling, counts of label 2: {}'.format(sum(y_train_sm == 2))) 
print('After oversampling, counts of label 3: {}'.format(sum(y_train_sm == 3)))<jupyter_output>After oversampling, the shape of train_X: (8430, 28)
After oversampling, the shape of train_y: (8430,) 

After oversampling, counts of label 1: 2810
After oversampling, counts of label 2: 2810
After oversampling, counts of label 3: 2810<jupyter_text>Use SMOTE data to fit new logistic regression model and run cross validation<jupyter_code>lr_sm = LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=1000) 
lr_sm.fit(X_train_sm, y_train_sm) 
scores_lr_sm = cross_val_score(lr_sm, X_train_sm, y_train_sm, cv=5, scoring=comb_metric, n_jobs=-1)
scores_lr_sm<jupyter_output><empty_output><jupyter_text>Use gridsearch to fit SMOTE data to random forest model and print best params and best scores<jupyter_code>grid_sm = GridSearchCV(rf_2, param_grid, iid=True, cv=5, n_jobs=-1,scoring=comb_metric, verbose=1, return_train_score=True)
grid_sm.fit(X_train_sm, y_train_sm)
print(grid_sm.best_params_)
print(grid_sm.best_score_)<jupyter_output>Fitting 5 folds for each of 20 candidates, totalling 100 fits<jupyter_text>Evaluate test performance Evaluate performance of the RF model found using grid search then applying SMOTE to it<jupyter_code>scores_rf = cross_validate(grid_sm, X_test, y_test, scoring=comb_metric, cv=5)
scores_rf['test_score'].mean()<jupyter_output>Fitting 5 folds for each of 20 candidates, totalling 100 fits<jupyter_text>Evaluate performance of logistic regression model w/o SMOTE on test data<jupyter_code>predictions_lr2 = lr_2.predict(X_test)
scores_lr2 = cross_validate(lr_2, X_test, y_test, scoring=comb_metric, cv=5)
round(scores_lr2['test_score'].mean(),3)<jupyter_output><empty_output><jupyter_text>Evaluate performance of logistic regression model with SMOTE on test data<jupyter_code>predictions_sm2 = lr_sm.predict(X_test)
scores_lr_sm = cross_validate(lr_sm, X_test, y_test, scoring=comb_metric, cv=5)
round(scores_lr_sm['test_score'].mean(),3)<jupyter_output><empty_output><jupyter_text>Pickle logistic regression model for use in web app with joblib<jupyter_code>joblib.dump(lr_2, '../src/models/clf_2.pkl', compress=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11485513046262207, 'fraction_numerical': 0.04618216744037138, 'mean_word_length': 6.928299492385787, 'pattern_counts': {'":': 1, '<': 114, '<?xml version=': 0, '>': 114, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '31', 'dir': 'jupyter-structured-clean-dedup', 'id': '101960', 'license': 'MIT', 'n_tokens_mistral': 4917, 'n_tokens_neox': 4438, 'n_words': 1040, 'path': 'notebooks/3-al-modeling.ipynb', 'repo_name': 'ali0003433/predict-vote-with-issue-import', 'size': '12494'}