<jupyter_start><jupyter_text>Baseline TestsThis notebook includes the classifiers that will be used to determine the performance of the deep learning-based classifier. Classifiers:1) Naive Bayes2) Support Vector Machines (SVM) Performance Measures:1) Storage requirements of the classifier and feature representation used2) Training time of the classifier3) Speed of the classifier4) Accuracy of the classifier<jupyter_code># nltk.download('punkt') #uncomment if running on new machine
# Importing the libraries
from sklearn import svm
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import gensim
from gensim.test.utils import datapath
from gensim.models import Word2Vec
import os
import nltk
from nltk import word_tokenize
from nltk import sent_tokenize

# Helpful variables
EXT_DATA_FOLDER = "C:\\Users\\Admin\\Projects\\thesis\\data\\"
EXT_DATA_FOLDER2 = "B:\\Datasets\\"

ANALYSIS_SAMPLES = os.path.join(EXT_DATA_FOLDER, "Credibility_Analysis_Samples\\September_25\\")
dataset_columns = ['Identifier', 'Type', 'Category', 'URL', 'Cat1', 'Cat2', 'Cat3', 'Cat4', 'Cat5',
 'Cat6', 'Cat7', 'Score', 'First date_time', 'Tweets', 'Likes', 'Retweets',
 'Potential exposure', 'HTML', 'TEXT']
criterias = ["Cat1", "Cat2", "Cat3", "Cat4", "Cat5", "Cat6", "Cat7"]<jupyter_output>C:\Users\Admin\Anaconda3\lib\site-packages\gensim\utils.py:1209: UserWarning: detected Windows; aliasing chunkize to chunkize_serial
  warnings.warn("detected Windows; aliasing chunkize to chunkize_serial")<jupyter_text>Reading in csv and excel data<jupyter_code>def create_dataset(corpus_path, annotated_samples):
    """
    Input: 
    corpus_path: Path for a CSV file containing a list of article URLs and its article text
    annotated_samples: Path of the excel file containing articles and its associated URL along with its labels
    
    Method:
    Retrieves the article text by matching the URLs within the corpus_path and annotated_samples and creates a dataframe 
    containing the URL, article text and the article's corresponding labels.
    
    Output:
    A pandas dataframe
    """
    article_corpus = pd.read_csv(corpus_path)
    annotated_corpus = pd.read_excel((annotated_samples))
    article_corpus.columns = ["URL", "HTML", "TEXT"]
    annotated_articles = annotated_corpus.loc[(annotated_corpus["Cat7"] == 0) | (annotated_corpus["Cat7"] == 1)]
    dataset = pd.merge(annotated_articles, article_corpus, how='left', on='URL')
    return dataset

article_corpus = pd.read_csv(corpus_path)
article_corpus.columns = ["URL", "HTML", "TEXT"]
print(article_corpus.shape)

print(article_corpus.loc[article_corpus["URL"] == "https://www.thestar.com/news/world/2017/05/07/anti-vaccine-activists-just-sparked-a-us-states-worst-measles-outbreak-in-decades.html"]["TEXT"])
print(article_corpus.loc[article_corpus["URL"] == "https://www.newscientist.com/article/mg23531335-800-cancer-vaccines-could-prime-our-own-bodies-to-fight-tumours/?utm_campaign=RSS%7CNSNS&utm_source=NSNS&utm_medium=RSS&campaign_id=RSS%7CNSNS-"])
corpus_path = os.path.join(EXT_DATA_FOLDER, "url_text.csv")
excel_files = ["sample_third_adam_new.xlsx", "sample_third_amalie_new.xlsx", "sample_third_maryke_new.xlsx"]

df_files = []

for filename in excel_files:
    annotated_path = os.path.join(ANALYSIS_SAMPLES, filename)
    data = create_dataset(corpus_path, annotated_path)
    df_files.append(data)
    
dataset = pd.concat(df_files)

print(dataset.columns.values)
print(dataset.shape)
for filename in excel_files[1:]:
    print(filename)
#Example of article with missing text
print(dataset.head()["URL"][3])
print(dataset.head()["TEXT"][3])  
print(dataset.head()["HTML"])
#Save dataset locally
writer = pd.ExcelWriter("dataset3.xlsx")
dataset.to_excel(writer, "Sheet1")
writer.save()
#pre-processing
from collections import defaultdict

labelled_articles = pd.read_excel("dataset5.xlsx")
labelled_articles = labelled_articles.dropna(subset=['TEXT'])
labelled_articles = labelled_articles[pd.to_numeric(labelled_articles['Cat1'], errors='coerce').notnull()]

for criteria in criterias:
    labelled_articles = labelled_articles.dropna(subset=[criteria])

print(labelled_articles.shape)
art_text_sent = np.array([sent_tokenize(article.split("TITLE: ")[1].replace("TEXT: ","").strip(" ")) for article in labelled_articles["TEXT"]])
art_text_word = np.array([word_tokenize(article.split("TITLE: ")[1].replace("TEXT: ","").strip(" ")) for article in labelled_articles["TEXT"]])
art_text_sent_word = np.array([[word_tokenize(sent) for sent in article] for article in art_text_sent])
labels = [labelled_articles["Cat1"], labelled_articles["Cat2"], labelled_articles["Cat3"], labelled_articles["Cat4"], labelled_articles["Cat5"], labelled_articles["Cat6"], labelled_articles["Cat7"]]
labels = np.array(labels).transpose()
multi_labels = np.array([[int(x) for x in row] for row in labels])<jupyter_output>(470, 21)<jupyter_text>Baseline classifier performancePerformance of classifier is measured using an f1_micro score:'f1_micro': Calculate metrics globally by counting the total true positives, false negatives and false positives and accounts for class imbalance. [Source](https://stackoverflow.com/questions/43421456/computing-macro-f1-score-using-sklearn)f1_micro scores are calculated using stratified k-fold cross validation for k = 10<jupyter_code>from sklearn import metrics
from sklearn.metrics import f1_score
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.pipeline import Pipeline
from sklearn.linear_model import SGDClassifier
from sklearn.externals import joblib
import timeit

import warnings
warnings.filterwarnings('ignore')

categories = ['Not Satisfied', 'Satisfied']
nb_bow = []
nb_tfidf = []
nb_w2v = []
svm_bow = []
svm_tfidf = []
svm_w2v = []

#NB Parameters

nb_parameters = {  
    'vec__max_df': (0.5, 0.625, 0.75, 0.875, 1.0),  
    'vec__max_features': (None, 5000, 10000, 20000),  
    'vec__min_df': (1, 5, 10, 20, 50),  
    'tfidf__use_idf': (True, False),  
    'tfidf__sublinear_tf': (True, False),  
    'vec__binary': (True, False),  
    'tfidf__norm': ('l1', 'l2'),  
    'clf__alpha': [10 ** x for x in range(-5, 1)]
    }

#SVM Parameters 

svm_parameters = {  
    'vec__max_df': (0.5, 0.625, 0.75, 0.875, 1.0),  
    'vec__max_features': (None, 5000, 10000, 20000),  
    'vec__min_df': (1, 5, 10, 20, 50),  
    'tfidf__use_idf': (True, False),  
    'tfidf__sublinear_tf': (True, False),  
    'vec__binary': (True, False),  
    'tfidf__norm': ('l1', 'l2'),  
    'clf__penalty': [None, 'l1', 'l2', 'elasticnet'],
    'clf__alpha': [10 ** x for x in range(-5, 1)]  
    }

nb_optimal = {
    "Cat1": Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict',
        encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english')), 
        ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),
    "Cat2": Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict', 
        encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english')), 
        ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),
    "Cat3": Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict', 
        encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english')), 
        ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),
    "Cat4": Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict',
        encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english')), 
        ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),
    "Cat5": Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict',
        encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english')),
        ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),
    "Cat6": Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict',
        encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english')), 
        ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),
    "Cat7": Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict',
        encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english')), 
        ('clf', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])
}

svm_optimal = {
    "Cat1": None,
    "Cat2": None,
    "Cat3": None,
    "Cat4": None,
    "Cat5": None,
    "Cat6": None,
    "Cat7": None
}<jupyter_output><empty_output><jupyter_text>NB Performance using gridsearch results<jupyter_code>for criteria in criterias:
    
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = nb_optimal[criteria]
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))
    #print("Actual vs. NB Predicted labels:\n" + str(y_test) + "\n" + str(nb_predicted))

    #print(metrics.classification_report(y_test, nb_predicted, target_names=categories))

    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    print("For " + criteria + ":")
    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
"""
    svm_clf = svm_optimal[criteria]

    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    #print("Actual vs. SVM Predicted labels:\n" + str(y_test) + "\n" + str(svm_predicted))

    #print(metrics.classification_report(y_test, svm_predicted, target_names=categories))

    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    print("SVM Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))
    svm_bow.append((svm_cv_scores.mean(), svm_cv_scores.std()))

parameters = {'vect__ngram_range': [(1, 1), (1, 2)],
              'tfidf__use_idf': (True, False),
              'clf__alpha': (1e-2, 1e-3),}
"""

#print(nb_f1_scores)
#print("====")
#print(svm_f1_scores)<jupyter_output>For Cat1:
NB Average micro f1-score: 0.85 (+/- 0.05)
For Cat2:
NB Average micro f1-score: 0.77 (+/- 0.05)
For Cat3:
NB Average micro f1-score: 0.86 (+/- 0.03)
For Cat4:
NB Average micro f1-score: 0.88 (+/- 0.04)
For Cat5:
NB Average micro f1-score: 0.65 (+/- 0.09)
For Cat6:
NB Average micro f1-score: 0.78 (+/- 0.06)
For Cat7:
NB Average micro f1-score: 0.83 (+/- 0.03)<jupyter_text>BoW performance with stopwords<jupyter_code>times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(max_features=20000)),
                         #('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))

    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_bow_stop_' + criteria + '.joblib')
    times.append(elapsed)
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()

    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer(max_features=20000)),
                         #('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, random_state=69,
                                               max_iter=5, tol=None)),
    ])
    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("SVM Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(svm_clf, 'models/svm_clf_bow_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
print(times)<jupyter_output>[]<jupyter_text>BoW Performance without stopwords<jupyter_code>times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word', max_features=20000)),
                         #('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))
    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_bow_no_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()

    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word', max_features=20000)),
                         #('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, random_state=69,
                                               max_iter=5, tol=None)),
    ])
    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("SVM Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(svm_clf, 'models/svm_clf_bow_no_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))<jupyter_output>Training Classifier for Cat1:
SVM Average micro f1-score: 0.79 (+/- 0.06)
Elapsed: 4.873469679617301 (seconds)
Training Classifier for Cat2:
SVM Average micro f1-score: 0.76 (+/- 0.06)
Elapsed: 4.882284293417797 (seconds)
Training Classifier for Cat3:
SVM Average micro f1-score: 0.77 (+/- 0.10)
Elapsed: 4.841570828020849 (seconds)
Training Classifier for Cat4:
SVM Average micro f1-score: 0.83 (+/- 0.06)
Elapsed: 4.9594816950202585 (seconds)
Training Classifier for Cat5:
SVM Average micro f1-score: 0.61 (+/- 0.06)
Elapsed: 4.748920988081409 (seconds)
Training Classifier for Cat6:
SVM Average micro f1-score: 0.76 (+/- 0.04)
Elapsed: 4.809942198541648 (seconds)
Training Classifier for Cat7:
SVM Average micro f1-score: 0.80 (+/- 0.03)
Elapsed: 4.948428662349215 (seconds)
Average time amongst all criteria: 4.866299763578354<jupyter_text>TF-IDF Performance with stopwords<jupyter_code>times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(max_features=20000)),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))

    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_tfidf_stop_' + criteria + '.joblib') 
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()

    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer(max_features=20000)),
                         ('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, random_state=69,
                                               max_iter=5, tol=None)),
    ])
    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("SVM Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(svm_clf, 'models/svm_clf_tfidf_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()

    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer(max_features=10)),
                         ('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, random_state=69,
                                               max_iter=5, tol=None)),
    ])
    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("SVM Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(svm_clf, 'models/svm_clf_tfidf_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))<jupyter_output>Training Classifier for Cat1:
SVM Average micro f1-score: 0.76 (+/- 0.02)
Elapsed: 6.442632911552437 (seconds)
Training Classifier for Cat2:
SVM Average micro f1-score: 0.62 (+/- 0.04)
Elapsed: 7.034491640694682 (seconds)
Training Classifier for Cat3:
SVM Average micro f1-score: 0.87 (+/- 0.01)
Elapsed: 4.754610664845131 (seconds)
Training Classifier for Cat4:
SVM Average micro f1-score: 0.79 (+/- 0.01)
Elapsed: 4.54878556097367 (seconds)
Training Classifier for Cat5:
SVM Average micro f1-score: 0.52 (+/- 0.06)
Elapsed: 4.755904096432914 (seconds)
Training Classifier for Cat6:
SVM Average micro f1-score: 0.77 (+/- 0.01)
Elapsed: 6.06543878896241 (seconds)
Training Classifier for Cat7:
SVM Average micro f1-score: 0.84 (+/- 0.00)
Elapsed: 5.71763290167587 (seconds)
Average time amongst all criteria: 5.617070937876731<jupyter_text>TF-IDF Performance without stopwords<jupyter_code>times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word', max_features=20000)),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))
    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_tfidf_no_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word', max_features=10000)),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))
    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_tfidf_no_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word', max_features=5000)),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))
    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_tfidf_no_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word')),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))
    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_tfidf_no_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word', max_features=1)),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)
    nb_predicted = list(nb_clf.predict(X_test))
    cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time

    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (cv_scores.mean(), cv_scores.std()))
    nb_bow.append((cv_scores.mean(), cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_tfidf_no_stop_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))<jupyter_output>Training Classifier for Cat1:
NB Average micro f1-score: 0.74 (+/- 0.01)
Elapsed: 5.143675597560275 (seconds)
Training Classifier for Cat2:
NB Average micro f1-score: 0.64 (+/- 0.01)
Elapsed: 7.642394215350464 (seconds)
Training Classifier for Cat3:
NB Average micro f1-score: 0.86 (+/- 0.01)
Elapsed: 6.620976347268595 (seconds)
Training Classifier for Cat4:
NB Average micro f1-score: 0.79 (+/- 0.01)
Elapsed: 6.064434937507201 (seconds)
Training Classifier for Cat5:
NB Average micro f1-score: 0.52 (+/- 0.01)
Elapsed: 4.862095461888202 (seconds)
Training Classifier for Cat6:
NB Average micro f1-score: 0.77 (+/- 0.01)
Elapsed: 5.0028922457709655 (seconds)
Training Classifier for Cat7:
NB Average micro f1-score: 0.84 (+/- 0.01)
Elapsed: 6.14640310265554 (seconds)
Average time amongst all criteria: 5.926124558285892<jupyter_text>LinearSVC performance using BoW/TFIDF with and without stopwords  LinearSVC + BoW with stopwords<jupyter_code>from sklearn.svm import LinearSVC

for criteria in criterias:
    print("For " + criteria + ":")
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer()),
                         #('tfidf', TfidfTransformer()),
                         ('clf', LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,
     intercept_scaling=1, loss='squared_hinge', max_iter=1000,
     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,
     verbose=0)),
    ])

    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    #print("Actual vs. LinearSVC Predicted labels:\n" + str(y_test) + "\n" + str(svm_predicted))

    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    print("LinearSVC Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))<jupyter_output>For Cat1:
LinearSVC Average micro f1-score: 0.80 (+/- 0.09)
For Cat2:
LinearSVC Average micro f1-score: 0.76 (+/- 0.06)
For Cat3:
LinearSVC Average micro f1-score: 0.88 (+/- 0.03)
For Cat4:
LinearSVC Average micro f1-score: 0.88 (+/- 0.07)
For Cat5:
LinearSVC Average micro f1-score: 0.76 (+/- 0.10)
For Cat6:
LinearSVC Average micro f1-score: 0.76 (+/- 0.10)
For Cat7:
LinearSVC Average micro f1-score: 0.82 (+/- 0.11)<jupyter_text>LinearSVC + TF-IDF with stopwords<jupyter_code>for criteria in criterias:
    print("For " + criteria + ":")
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer()),
                         ('tfidf', TfidfTransformer()),
                         ('clf', LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,
     intercept_scaling=1, loss='squared_hinge', max_iter=1000,
     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,
     verbose=0)),
    ])

    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    #print("Actual vs. LinearSVC Predicted labels:\n" + str(y_test) + "\n" + str(svm_predicted))

    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    print("LinearSVC Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))<jupyter_output>For Cat1:
LinearSVC Average micro f1-score: 0.84 (+/- 0.06)
For Cat2:
LinearSVC Average micro f1-score: 0.84 (+/- 0.06)
For Cat3:
LinearSVC Average micro f1-score: 0.89 (+/- 0.02)
For Cat4:
LinearSVC Average micro f1-score: 0.88 (+/- 0.04)
For Cat5:
LinearSVC Average micro f1-score: 0.78 (+/- 0.07)
For Cat6:
LinearSVC Average micro f1-score: 0.76 (+/- 0.07)
For Cat7:
LinearSVC Average micro f1-score: 0.82 (+/- 0.04)<jupyter_text>LinearSVC + BoW without stopwords<jupyter_code>for criteria in criterias:
    print("For " + criteria + ":")
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word')),
                         #('tfidf', TfidfTransformer()),
                         ('clf', LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,
     intercept_scaling=1, loss='squared_hinge', max_iter=1000,
     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,
     verbose=0)),
    ])

    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    #print("Actual vs. LinearSVC Predicted labels:\n" + str(y_test) + "\n" + str(svm_predicted))

    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    print("LinearSVC Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))<jupyter_output>For Cat1:
LinearSVC Average micro f1-score: 0.83 (+/- 0.08)
For Cat2:
LinearSVC Average micro f1-score: 0.83 (+/- 0.09)
For Cat3:
LinearSVC Average micro f1-score: 0.90 (+/- 0.04)
For Cat4:
LinearSVC Average micro f1-score: 0.87 (+/- 0.06)
For Cat5:
LinearSVC Average micro f1-score: 0.80 (+/- 0.09)
For Cat6:
LinearSVC Average micro f1-score: 0.80 (+/- 0.11)
For Cat7:
LinearSVC Average micro f1-score: 0.79 (+/- 0.06)<jupyter_text>LinearSVC + TF-IDF without stopwords<jupyter_code>for criteria in criterias:
    print("For " + criteria + ":")
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_clf = Pipeline([('vect', CountVectorizer(stop_words='english', analyzer='word')),
                         ('tfidf', TfidfTransformer()),
                         ('clf', LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,
     intercept_scaling=1, loss='squared_hinge', max_iter=1000,
     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,
     verbose=0)),
    ])

    svm_clf.fit(X_train, y_train)
    svm_predicted = list(svm_clf.predict(X_test))
    #print("Actual vs. LinearSVC Predicted labels:\n" + str(y_test) + "\n" + str(svm_predicted))

    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    print("LinearSVC Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))<jupyter_output>For Cat1:
LinearSVC Average micro f1-score: 0.80 (+/- 0.07)
For Cat2:
LinearSVC Average micro f1-score: 0.82 (+/- 0.03)
For Cat3:
LinearSVC Average micro f1-score: 0.90 (+/- 0.02)
For Cat4:
LinearSVC Average micro f1-score: 0.86 (+/- 0.04)
For Cat5:
LinearSVC Average micro f1-score: 0.76 (+/- 0.02)
For Cat6:
LinearSVC Average micro f1-score: 0.75 (+/- 0.09)
For Cat7:
LinearSVC Average micro f1-score: 0.81 (+/- 0.04)<jupyter_text>Helper functions for calculating mean of word vectors<jupyter_code>class MeanEmbeddingVectorizer(object):
    def __init__(self, word2vec):
        self.word2vec = word2vec
        # if a text is empty we should return a vector of zeros
        # with the same dimensionality as all the other vectors
        self.dim = len(word2vec.items())

    def fit(self, X, y):
        return self

    def transform(self, X):
        return np.array([
            np.mean([self.word2vec[w] for w in words if w in self.word2vec]
                    or [np.zeros(self.dim)], axis=0)
            for words in X
        ])
class TfidfEmbeddingVectorizer(object):
    def __init__(self, word2vec):
        self.word2vec = word2vec
        self.word2weight = None
        self.dim = len(word2vec.items())

    def fit(self, X, y):
        tfidf = TfidfVectorizer(analyzer=lambda x: x)
        tfidf.fit(X)
        # if a word was never seen - it must be at least as infrequent
        # as any of the known words - so the default idf is the max of 
        # known idf's
        max_idf = max(tfidf.idf_)
        self.word2weight = defaultdict(
            lambda: max_idf,
            [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()])

        return self

    def transform(self, X):
        return np.array([
                np.mean([self.word2vec[w] * self.word2weight[w]
                         for w in words if w in self.word2vec] or
                        [np.zeros(self.dim)], axis=0)
                for words in X
            ])
with open(os.path.join(EXT_DATA_FOLDER, "glove.6B.300d.txt"), "rb") as lines:
    w2v = {line.split()[0]: np.array(map(float, line.split()[1:]))
           for line in lines}
w2v_df = pd.DataFrame.from_dict(w2v, orient='index')
w2v_df.head()[0]<jupyter_output><empty_output><jupyter_text>Word2Vec with Mean Embedding Vectorizer<jupyter_code>from gensim.sklearn_api import W2VTransformer
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    print("For " + criteria + ":")
    
    
    nb_clf = Pipeline([('w2v', MeanEmbeddingVectorizer(w2v)),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)

    nb_predicted = nb_clf.predict(X_test)
    nb_cv_scores = cross_val_score(nb_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time
    
    
    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (nb_cv_scores.mean(), nb_cv_scores.std()))
    nb_w2v.append((nb_cv_scores.mean(), nb_cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(nb_clf, 'models/nb_clf_glove_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))

from gensim.sklearn_api import W2VTransformer
times = []
for criteria in criterias:
    print("Training Classifier for " + criteria + ":")
    start_time = timeit.default_timer()
    
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))
    
    svm_clf = Pipeline([('w2v', MeanEmbeddingVectorizer(w2v)),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, random_state=69,
                                               max_iter=5, tol=None)),
    ])
    
    svm_clf.fit(X_train, y_train)
    svm_predicted = svm_clf.predict(X_test)
    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, cv=10, scoring='f1_micro')
    elapsed = timeit.default_timer() - start_time
    print("SVM Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))
    svm_w2v.append((svm_cv_scores.mean(), svm_cv_scores.std()))
    print("Elapsed: " + str(elapsed) + " (seconds)")
    joblib.dump(svm_clf, 'models/svm_clf_glove_' + criteria + '.joblib')
    times.append(elapsed)
    
average = np.array(times).mean()
print("Average time amongst all criteria: " + str(average))<jupyter_output>Training Classifier for Cat1:
SVM Average micro f1-score: 0.74 (+/- 0.01)
Elapsed: 3530.65862255462 (seconds)
Training Classifier for Cat2:
SVM Average micro f1-score: 0.64 (+/- 0.00)
Elapsed: 3420.329507030321 (seconds)
Training Classifier for Cat3:
SVM Average micro f1-score: 0.86 (+/- 0.01)
Elapsed: 3580.485808795729 (seconds)
Training Classifier for Cat4:
SVM Average micro f1-score: 0.79 (+/- 0.01)
Elapsed: 3694.0809074248245 (seconds)
Training Classifier for Cat5:
SVM Average micro f1-score: 0.51 (+/- 0.02)
Elapsed: 3565.754687701854 (seconds)
Training Classifier for Cat6:<jupyter_text>Word2Vec with Tf-idf Weighted Vectorizer<jupyter_code>from gensim.sklearn_api import W2VTransformer

for criteria in criterias:
    
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    print("For " + criteria + ":")
    
    
    nb_clf = Pipeline([('w2v', TfidfEmbeddingVectorizer(w2v)),
                         ('clf', MultinomialNB()),])
    nb_clf.fit(X_train, y_train)

    nb_predicted = nb_clf.predict(X_test)
    #print(metrics.classification_report(y_test, nb_predicted, target_names=categories))
    nb_cv_scores = cross_val_score(nb_clf, X_train, y_train, scoring='f1_micro')
    
    
    print("NB Average micro f1-score: %0.2f (+/- %0.2f)" % (nb_cv_scores.mean(), nb_cv_scores.std()))
    nb_w2v.append((nb_cv_scores.mean(), nb_cv_scores.std()))

    
    svm_clf = Pipeline([('w2v', TfidfEmbeddingVectorizer(w2v)),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, random_state=69,
                                               max_iter=5, tol=None)),
    ])
    
    svm_clf.fit(X_train, y_train)
    svm_predicted = svm_clf.predict(X_test)
    svm_cv_scores = cross_val_score(svm_clf, X_train, y_train, scoring='f1_micro')
    print("SVM Average micro f1-score: %0.2f (+/- %0.2f)" % (svm_cv_scores.mean(), svm_cv_scores.std()))
    svm_w2v.append((svm_cv_scores.mean(), svm_cv_scores.std()))
    

print("type of twent_test.data: ", type(twenty_train.target))
print(len(art_text))
print(twenty_test.target)
def load_embeddings(filename):
    """
    Load a DataFrame from the generalized text format used by word2vec, GloVe,
    fastText, and ConceptNet Numberbatch. The main point where they differ is
    whether there is an initial line with the dimensions of the matrix.
    """
    labels = []
    rows = []
    with open(filename, encoding='utf-8') as infile:
        for i, line in enumerate(infile):
            items = line.rstrip().split(' ')
            if len(items) == 2:
                # This is a header row giving the shape of the matrix
                continue
            labels.append(items[0])
            values = np.array([float(x) for x in items[1:]], 'f')
            rows.append(values)
    
    arr = np.vstack(rows)
    return pd.DataFrame(arr, index=labels, dtype='f')
#Loading pre-trained word2vec
pre_word2vec_model = gensim.models.KeyedVectors.load_word2vec_format(datapath(os.path.join(EXT_DATA_FOLDER, "GoogleNews-vectors-negative300.bin")), binary=True)
pre_word2vec_model.save("pre_word2vec.model")
print("Similarity of 'woman' and 'man': ", pre_word2vec_model.similarity('woman', 'man'))
print("Similarity of 'woman' and 'woman': ", pre_word2vec_model.similarity('woman', 'woman'))
print("Similarity of 'dog' and 'hotdog': ", pre_word2vec_model.similarity('dog', 'hotdog'))<jupyter_output>Similarity of 'woman' and 'man':  0.76640123
Similarity of 'woman' and 'woman':  1.0
Similarity of 'dog' and 'hotdog':  0.38931656<jupyter_text>Using Gridsearch to find optimal hyperparameters<jupyter_code>#NB Parameters

nb_parameters = {  
    'vec__max_df': (0.5, 0.625, 0.75, 0.875, 1.0),  
    'vec__max_features': (None, 5000, 10000, 20000),  
    'vec__min_df': (1, 5, 10, 20, 50),  
    'tfidf__use_idf': (True, False),  
    'tfidf__sublinear_tf': (True, False),  
    'vec__binary': (True, False),  
    'tfidf__norm': ('l1', 'l2'),  
    'clf__alpha': [10 ** x for x in range(-5, 1)]
    }

#SVM Parameters 

svm_parameters = {  
    'vec__max_df': (0.5, 0.625, 0.75, 0.875, 1.0),  
    'vec__max_features': (None, 5000, 10000, 20000),  
    'vec__min_df': (1, 5, 10, 20, 50),  
    'tfidf__use_idf': (True, False),  
    'tfidf__sublinear_tf': (True, False),  
    'vec__binary': (True, False),  
    'tfidf__norm': ('l1', 'l2'),  
    'clf__penalty': [None, 'l1', 'l2', 'elasticnet'],
    'clf__alpha': [10 ** x for x in range(-5, 1)]  
    }

best_model = defaultdict(str)
for criteria in criterias:
    print("Currently finding best model for: " + criteria)
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_pipeline = Pipeline([('vec', CountVectorizer(stop_words='english', analyzer='word')),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_grid_search = GridSearchCV(estimator=nb_pipeline, param_grid=nb_parameters, cv=10, n_jobs=-1, verbose=2)  
    nb_grid_search.fit(X_train, y_train)
    best_model[criteria] = nb_grid_search
    #nb_predicted = list(nb_grid_search.predict(X_test))
   
best_model['Cat1'] = nb_grid_search
print(criterias[1:])
for criteria in criterias[1:]:
    print("Currently finding best model for: " + criteria)
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    nb_pipeline = Pipeline([('vec', CountVectorizer(stop_words='english', analyzer='word')),
                         ('tfidf', TfidfTransformer()),
                         ('clf', MultinomialNB()),])
    nb_grid_search = GridSearchCV(estimator=nb_pipeline, param_grid=nb_parameters, cv=10, n_jobs=-1, verbose=2)  
    nb_grid_search.fit(X_train, y_train)
    best_model[criteria] = nb_grid_search
    #nb_predicted = list(nb_grid_search.predict(X_test))
svm_best_model = defaultdict(str)
svm_parameters = {  
    'vec__max_df': (0.5, 0.625, 0.75, 0.875, 1.0),  
    'vec__max_features': (None, 5000, 10000, 20000),  
    'vec__min_df': (1, 5, 10, 20, 50),  
    'tfidf__use_idf': (True, False),  
    'tfidf__sublinear_tf': (True, False),  
    'vec__binary': (True, False),  
    'tfidf__norm': ('l1', 'l2'),  
    'clf__penalty': [None, 'l1', 'l2', 'elasticnet'],
    'clf__alpha': [10 ** x for x in range(-5, 1)]  
    }
for criteria in criterias:
    print("Currently finding best model for: " + criteria)
    X_train, X_test, y_train, y_test = train_test_split(list(labelled_articles["TEXT"]), list(labelled_articles[criteria]), test_size=int(20))

    svm_pipeline = Pipeline([('vec', CountVectorizer(stop_words='english', analyzer='word')),
                         ('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier()),])
    svm_grid_search = GridSearchCV(estimator=svm_pipeline, param_grid=svm_parameters, cv=10, n_jobs=-1, verbose=2)  
    svm_grid_search.fit(X_train, y_train)
    svm_best_model[criteria] = svm_grid_search

print("========================== NB ==========================")
print(best_model)
print("========================== SVM ==========================")
print(svm_best_model)
for key, value in svm_best_model.items():
    print("For: " + key)
    print(value)
    print("=======================")<jupyter_output>For: Cat1
GridSearchCV(cv=10, error_score='raise',
       estimator=Pipeline(memory=None,
     steps=[('vec', CountVectorizer(analyzer='word', binary=False, decode_error='strict',
        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',
        lowercase=True, max_df=1.0, max_features=None, min_df=1,
        ngram_range=(1, 1), preprocessor=None, stop_words='english',
        s...='l2', power_t=0.5, random_state=None,
       shuffle=True, tol=None, verbose=0, warm_start=False))]),
       fit_params=None, iid=True, n_jobs=-1,
       param_grid={'vec__max_df': (0.5, 0.625, 0.75, 0.875, 1.0), 'vec__max_features': (None, 5000, 10000, 20000), 'vec__min_df': (1, 5, 10, 20, 50), 'tfidf__use_idf': (True, False), 'tfidf__sublinear_tf': (True, False), 'vec__binary': (True, False), 'tfidf__norm': ('l1', 'l2'), 'clf__penalty': [None, 'l1', 'l2', 'elasticnet'], 'clf__alpha': [1e-05, 0.0001, 0.001, 0.01, 0.1, 1]},
       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',
[...]
{'fraction_non_alphanumeric': 0.15216514112175336, 'fraction_numerical': 0.03977743326183448, 'mean_word_length': 4.009894669645707, 'pattern_counts': {'":': 35, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '588285', 'license': 'MIT', 'n_tokens_mistral': 18548, 'n_tokens_neox': 16799, 'n_words': 3344, 'path': 'Project/Baseline.ipynb', 'repo_name': 'DioneJM/Honours-Thesis', 'size': '47087'}