<jupyter_start><jupyter_text>Basic Modeling - Project III  - Choosing the best model to hypertune<jupyter_code># Import Basics
import numpy as np
import pandas as pd

#Import Visuals
import matplotlib.pyplot as plt
import seaborn as sns
import missingno as msno

# Import Linear Regression Algorithms
from sklearn.linear_model import LinearRegression, Lasso, LassoCV ,Ridge, ElasticNet, LogisticRegression, RidgeCV
from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier

from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn.ensemble import AdaBoostRegressor, AdaBoostClassifier

# Import Optimization algorithms
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.svm import SVR

# Import Data Processing
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder, PowerTransformer, PolynomialFeatures

from sklearn.impute import KNNImputer
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer, make_column_transformer, make_column_selector


from sklearn.naive_bayes import MultinomialNB

# NPL 
from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer


#scrapping 
import json

#Import graphical user info
from sklearn import set_config

# Metrics
from sklearn.metrics import accuracy_score, mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import balanced_accuracy_score
from sklearn.metrics import f1_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score

from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import confusion_matrix<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>"""
   Function to print out score from a make_pipeline flows
       input is list of the make_pipeline var name 
"""
score_train_list = []
score_test_list = []
accuracy_score_ = []
y_preds_list = []
precision_score_ = []
recall_score_ = []
f1_score_ = []


def score_pipline(the_pipes, all_pipes_name ):
    for est,name in zip(the_pipes,all_pipes_name):
       # fit model 
        est.fit(X_train, y_train)
       # score model  
        score_test = est.score(X_test, y_test)
        score_train = est.score(X_train, y_train)
       # create preds
        y_preds= est.predict(X_test)
        accuracy_score_.append(accuracy_score(y_test, y_preds))
        precision_score_.append(precision_score(y_test, y_preds))
        recall_score_.append(recall_score(y_test, y_preds))
        f1_score_.append(f1_score(y_test, y_preds))
       # create list to export 
        score_train_list.append(score_train)
        score_test_list.append(score_test)
        y_preds_list.append(y_preds)
       # output
        print()
        print(f'--  Confusion Maxtix --   {name}')
        print(confusion_matrix(y_test, y_preds)) 
        
    data = {
        'model': all_pipes_name,
        'train score':score_train_list,
        'test score':score_test_list,
        #'y_preds':y_preds_list,
        'accuracy score': accuracy_score_,
        'precision score': precision_score_,
        'recall score' : recall_score_,
        #'f1 score':f1_score_,  
    }
    df = pd.DataFrame(data)
    #df.set_index('model', inplace = True, drop = True)
    return df<jupyter_output><empty_output><jupyter_text>Import DataFrame<jupyter_code>df = pd.read_csv('../clean_data/clean_data.csv')
df.shape<jupyter_output><empty_output><jupyter_text>Running tests on a sample of the data<jupyter_code>"""
    this creates a random sample of the data to run test on our X and y 
        before we use all the data 
"""

df_sample = df.sample(2500).copy()
df_sample.head()<jupyter_output><empty_output><jupyter_text>---  Define X and y<jupyter_code>#    ---_sample
X = df['max_text']
X.shape
# starwars = 1 -- _sample--
y = np.where(df['subreddit'] == 'starwars', 1, 0)
y
y.shape
df_sample[(df_sample['subreddit'] == 'starwars')] #.value_counts(normalize = True)<jupyter_output><empty_output><jupyter_text>---  Train test split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, stratify =y , random_state =33)<jupyter_output><empty_output><jupyter_text>baseline / Null score<jupyter_code>1- y_train.sum()/len(y_train)<jupyter_output><empty_output><jupyter_text>Estimators & Vectize<jupyter_code>cvect = CountVectorizer(stop_words = 'english')
tfidf = TfidfVectorizer(stop_words = 'english')

nbayes = MultinomialNB()
clf = LogisticRegression()
forest = RandomForestClassifier()
ada = AdaBoostClassifier(n_estimators=100)<jupyter_output><empty_output><jupyter_text>Pipelines<jupyter_code>cvect_nbayes = make_pipeline(cvect, nbayes)
tfidf_nbayes = make_pipeline(tfidf, nbayes)

cvect_clf = make_pipeline(cvect, clf)
tfidf_clf = make_pipeline(tfidf, clf)

cvect_forest = make_pipeline(cvect, forest)
tfidf_forest = make_pipeline(tfidf, forest)

cvect_ada = make_pipeline(cvect, ada)
tfidf_ada = make_pipeline(tfidf, ada)
# list of pipelines

all_pipes = [cvect_nbayes, tfidf_nbayes,cvect_clf, tfidf_clf,cvect_forest, tfidf_forest,cvect_ada,tfidf_ada]
all_pipes_name = ['cvect_nbayes', 'tfidf_nbayes',
                  'cvect_clf', 'tfidf_clf',
                  'cvect_forest', 'tfidf_forest',
                  'cvect_ada','tfidf_ada']<jupyter_output><empty_output><jupyter_text>Fit & Score<jupyter_code>""""
    score  - a loop to compare out of the box performence estimators
        - each estimator uses both TfidfVectorizer & CountVectorizer
        - testing 4 separate estimators
"""

df_results = score_pipline(all_pipes, all_pipes_name)
# data frame with results
#df_results.set_index('model', inplace = True, drop = True)
#df_results = df_results.T

df_results<jupyter_output><empty_output><jupyter_text>---  Model Evaluation: In desided on our basic model we ran a base model of Naive Bayes, Logistic Regression, Random Forest and Adaboost classifier models, each paired with TfidfVectorizer and CountVectorizer.  We are looking to see which model has the best Accuracy and Precision.Specifically of the subreddits that we idenified as being Starwas, how many of those did we get correct. Using this Rubric we found that Naive Bayes with Count Vectorizer yeilds the highest performence. Below are all graphs showing each model accuracy and precision score.<jupyter_code># Graph for Accuracy 

plt.figure(figsize = (9,6))
ax = sns.barplot(x = 'model', y='accuracy score',
                 data = df_results,
                 palette="rocket", 
                 order=df_results.sort_values('accuracy score', ascending = False).model, alpha = .9,
                )
for p in ax.patches:
    ax.annotate(format(p.get_height(), '.4f'),
                  (p.get_x()+ p.get_width() / 2., p.get_height()),
                  ha = 'center', va = 'center',
                  xytext = (0,9),
                  textcoords = 'offset points')
plt.xlabel("Model", size=15)
plt.ylabel("Score", size=15)
plt.title("Accuracy Score", size=18)
plt.tight_layout()


plt.savefig("../clean_data/basic_model_Accuracy_Score", dpi=100)
# Graph for Test score 

plt.figure(figsize = (9,6))
ax = sns.barplot(x = 'model', y='precision score',
                 data = df_results,
                 palette="viridis",
                 order=df_results.sort_values('precision score', ascending = False).model, alpha = .9,
                )
for p in ax.patches:
    ax.annotate(format(p.get_height(), '.4f'),
                  (p.get_x()+ p.get_width() / 2., p.get_height()),
                  ha = 'center', va = 'center',
                  xytext = (0,9),
                  textcoords = 'offset points')
plt.xlabel("Model", size=15)
plt.ylabel("Score", size=15)
plt.title("Precision Score", size=18)
plt.tight_layout()

plt.savefig("../clean_data/basic_model_Precision_Score", dpi=100)
# Graph for Test score 

plt.figure(figsize = (9,6))
ax = sns.barplot(x = 'model', y='recall score',
                 data = df_results,
                 palette="viridis",
                 order=df_results.sort_values('recall score', ascending = False).model, alpha = .9,
                )
for p in ax.patches:
    ax.annotate(format(p.get_height(), '.4f'),
                  (p.get_x()+ p.get_width() / 2., p.get_height()),
                  ha = 'center', va = 'center',
                  xytext = (0,9),
                  textcoords = 'offset points')
plt.xlabel("Model", size=15)
plt.ylabel("Score", size=15)
plt.title("recall score", size=18)
plt.tight_layout()

plt.savefig("../clean_data/basic_model_Precision_Score", dpi=100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.103189919400613, 'fraction_numerical': 0.007719377908956749, 'mean_word_length': 3.9605855855855854, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '456031', 'license': 'MIT', 'n_tokens_mistral': 3098, 'n_tokens_neox': 2961, 'n_words': 668, 'path': 'code/02-basic_modeling-proj_3.ipynb', 'repo_name': 'encodingintuition/NPL_binary_classification-targeted_marketing_campaign.', 'size': '8809'}