<jupyter_start><jupyter_text>Model Building Part 3 Code for building the models  _Author: <NAME>_  _Email: <EMAIL>_   Following up with part one, I try the bag of words approach<jupyter_code>import os
import pandas as pd
import numpy as np
import scipy as sp
import seaborn as sns
import matplotlib.pyplot as plt
import json
from IPython.display import Image
from IPython.core.display import HTML
retval=os.chdir("..")
clean_data=pd.read_pickle('./clean_data/clean_data.pkl')
clean_data.head()
kept_cols=['helpful', 'text_lemma']<jupyter_output><empty_output><jupyter_text>Training and Testing Split<jupyter_code>my_rand_state=0
test_size=0.25
from sklearn.model_selection import train_test_split
X = (clean_data[kept_cols].iloc[:,1]).tolist()
y = (clean_data[kept_cols].iloc[:,0]).tolist()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, 
                                              random_state=my_rand_state)<jupyter_output><empty_output><jupyter_text>Text<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
tfidf=TfidfVectorizer(lowercase=False)<jupyter_output><empty_output><jupyter_text>Classification Models<jupyter_code>from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier<jupyter_output><empty_output><jupyter_text>Although tuning is not necessary for Naive Bayes, I pass the default parameters of those models to GridSearchCV anyway so that I can do a direct pair-wise comparison with the other models across the different steps of cross-validation.  In the interest of time, I didn't use the SVM classifier.<jupyter_code>nb_clf=GaussianNB()
priors=[None]
qda_clf=QuadraticDiscriminantAnalysis()
reg_param=[0.0, 0.25, 0.5, 0.75]
log_clf=LogisticRegression(penalty='l2')
C=[0.001 , 0.01, 10, 100,1000]
rf_clf=RandomForestClassifier()
n_estimators=[100,200]
max_features=[.1,.3,.5]
dtree=DecisionTreeClassifier(max_depth=None, min_samples_split=2)
bagTree_clf=BaggingClassifier(base_estimator=dtree)
max_samples=[.3,.6]
class_weight=['balanced']
class_weight.extend([{1: w} for w in [1, 2, 10]])<jupyter_output><empty_output><jupyter_text>Creating Pipelines<jupyter_code>from imblearn import pipeline #needed if mixing imblearn with sklearn classes
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold<jupyter_output><empty_output><jupyter_text>I plan on using imblearn classes for later iterations so I use it's pipeline in the beginning for convenience<jupyter_code>n_jobs=4
n_folds=10
skfold = StratifiedKFold(n_splits=n_folds,random_state=my_rand_state, shuffle=False)<jupyter_output><empty_output><jupyter_text>Naive Bayes Estimators<jupyter_code>nb_clf_b = pipeline.Pipeline(steps=[('tfidf',tfidf),('clf',nb_clf)])
nb_clf_est_b = GridSearchCV(estimator=nb_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__priors=priors))<jupyter_output><empty_output><jupyter_text>QDA Estimators<jupyter_code>qda_clf_b = pipeline.Pipeline(steps=[('tfidf',tfidf),('clf',qda_clf)])
qda_clf_est_b = GridSearchCV(estimator=qda_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__reg_param=reg_param))<jupyter_output><empty_output><jupyter_text>Logistic Estimators<jupyter_code>log_clf_b = pipeline.Pipeline(steps=[('tfidf',tfidf),('clf',log_clf)])
log_clf_est_b = GridSearchCV(estimator=log_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__C=C,
                              clf__class_weight=class_weight))<jupyter_output><empty_output><jupyter_text>Random Forest Estimators<jupyter_code>rf_clf_b = pipeline.Pipeline(steps=[('tfidf',tfidf),('clf',rf_clf)])
rf_clf_est_b = GridSearchCV(estimator=rf_clf_b,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__n_estimators=n_estimators,
                              clf__max_features=max_features,
                              clf__class_weight=class_weight))<jupyter_output><empty_output><jupyter_text>Fitting Estimators<jupyter_code>from sklearn.externals import joblib<jupyter_output><empty_output><jupyter_text>Basic Estimators: no bag of words or PCA<jupyter_code>log_clf_est_b.fit(X_train,y_train)
joblib.dump(log_clf_est_b, './other_output/bow/log_clf_est_b.pkl')
rf_clf_est_b.fit(X_train,y_train)
joblib.dump(rf_clf_est_b, './other_output/bow/rf_clf_est_b.pkl')<jupyter_output><empty_output><jupyter_text>Testing Estimators<jupyter_code>from sklearn.metrics import roc_curve, auc
log_clf_est_b=joblib.load('./other_output/bow/log_clf_est_b.pkl')
rf_clf_est_b=joblib.load('./other_output/bow/rf_clf_est_b.pkl')<jupyter_output><empty_output><jupyter_text>Basic Estimators: no bag of words or PCA<jupyter_code>log_fpr, log_tpr, _ = roc_curve(y_test, 
                    log_clf_est_b.predict_proba(X_test)[:,1])
log_roc_auc = auc(log_fpr, log_tpr)

rf_fpr, rf_tpr, _ = roc_curve(y_test, 
                    rf_clf_est_b.predict_proba(X_test)[:,1])
rf_roc_auc = auc(rf_fpr, rf_tpr)
plt.plot(log_fpr, log_tpr, color='seagreen', linestyle='--',
         label='LOG (area = %0.2f)' % log_roc_auc, lw=2)

plt.plot(rf_fpr, rf_tpr, color='blue', linestyle='--',
         label='RF (area = %0.2f)' % rf_roc_auc, lw=2)

plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='k',
         label='Luck')

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curves of Basic Models Using BOW')
plt.legend(loc="lower right")
plt.savefig('./plots/ROC_Basic_BOW.png', bbox_inches='tight')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11678832116788321, 'fraction_numerical': 0.013410286878288916, 'mean_word_length': 6.337484433374844, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '362885', 'license': 'MIT', 'n_tokens_mistral': 2351, 'n_tokens_neox': 2256, 'n_words': 345, 'path': "code/.ipynb_checkpoints/model_building_part_3-checkpoint (jimmy-Precision-T1600's conflicted copy 2017-05-14).ipynb", 'repo_name': 'ViniRa/thon', 'size': '5914'}