<jupyter_start><jupyter_text>Naive Bayes (NB), Logistic Regression (LR)*** Imports<jupyter_code>import numpy as np
import math
from xml.dom import minidom
from xml.etree import cElementTree as ElementTree
import os
import nltk
import pickle
import csv
import timeit
import re
import warnings
warnings.filterwarnings('ignore')
from scipy.sparse import vstack, hstack
from sklearn.datasets import make_classification
import pandas as pd
from sklearn.model_selection import train_test_split, cross_validate, cross_val_score
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import KFold
from sklearn.metrics import recall_score, make_scorer, precision_score, f1_score, accuracy_score
from sklearn.naive_bayes import MultinomialNB
from functools import partial
import matplotlib.pyplot as plt
import copy
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\diego\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\diego\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package wordnet to
[nltk_data]     C:\Users\diego\AppData\Roaming\nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>File reading<jupyter_code>def printExecTime(func):
    """
    Decorator definition for printing the execution time of a function. 
    :param func. Function for which to measure execution time.
    """
    def wrapper(*args, **kwargs):
        init_time = timeit.default_timer()
        result = func(*args, **kwargs)
        stop_time = timeit.default_timer()
        total = stop_time - init_time
        print('Execution time in sec for', func.__name__, 'was', total)
        return result
    return wrapper

CLASSES = {}

@printExecTime
def documentReader(path):
    global CLASSES
    documents_path = os.path.join(os.getcwd(), path)
    docs = {}
    id = 1
    for dir_name in os.listdir(documents_path):
        CLASSES[id] = dir_name
        CLASSES[dir_name] = id
        class_path = os.path.join(documents_path, dir_name)

        for document in os.listdir(class_path):
            document_path = os.path.join(class_path, document)
            file = open(document_path, 'r').read()
            docs[str(id) + '-' + document] = file.replace(u'\xa0', u' ').replace('\n', ' ')
        id += 1
    return docs

raw_docs = documentReader('datasets/20news')
print('Count', len(raw_docs), 'docs read')
print('CLASSES and ids', list(CLASSES.items())[:2])<jupyter_output>Execution time in sec for documentReader was 81.52083779999998
Count 18828 docs read
CLASSES and ids [(1, 'alt.atheism'), ('alt.atheism', 1)]<jupyter_text>Proccessing pipeline<jupyter_code>def regexPipeFilter(text):
    stop_words = set(nltk.corpus.stopwords.words("english"))
    lemmatizer = nltk.stem.WordNetLemmatizer()

    # Remove all the special characters
    document = re.sub(r'\W', ' ', text)

    # remove all single characters
    document = re.sub(r'\s+[a-zA-Z]\s+', ' ', document)

    # Converting to Lowercase
    document = document.lower()

    # Lemmatization
    document = document.split()

    document = [lemmatizer.lemmatize(word) for word in document if word not in stop_words]
    document = ' '.join(document)
    return document

@printExecTime
def proccessDocs(docs): 
    """
    Proccesing pipiling for the raw docs read from files
    :param docs: Dictionary containing the raw docs previously read from files
    :return: 
    """
    getClass = lambda x: int(x.split('-')[0])
    proccessed_docs = [[regexPipeFilter(document), getClass(key), key] for key, document in docs.items()]

    return proccessed_docs

tokenized_docs = proccessDocs(raw_docs)
print(tokenized_docs[0][1])
print('Class:', CLASSES[tokenized_docs[0][1]], '.', tokenized_docs[0][0][:100])<jupyter_output>1
Class: alt.atheism . mathew mathew mantis co uk subject alt atheism faq atheist resource archive name atheism resource al<jupyter_text>Dataset division<jupyter_code>@printExecTime
def divideSets(tokenized_docs,):
    train, test = 0.7, 0.3
    # test_ratio, dev_ratio = test/(dev+test), dev/(dev+test)
    total = len(tokenized_docs)

    train_set, test_set = train_test_split(tokenized_docs, test_size= test, train_size=train)
    # dev_set, test_set = train_test_split(rest, test_size=test_ratio, train_size= dev_ratio)

    print('Train', len(train_set)/total)
    print('Test', len(test_set)/total)
    # print('Dev', len(dev_set)/total)

    return train_set, test_set

train_set, test_set= divideSets(tokenized_docs)
def extractXY(dataset):
    """
    Extracts individual vectors for documents and document's classes.
    """
    x , y= [], []
    for [x_i,  y_i, _] in dataset:
        x.append(x_i)
        y.append(y_i)
    return x, y

train_x, train_y = extractXY(train_set)
# dev_x, dev_y = extractXY(dev_set)
test_x, test_y = extractXY(test_set)
print(train_y[0])
print(train_x[0][:100])<jupyter_output>15
cohen ssdgwy mdc com andy cohen subject dc news static test firing scheduled saturday many schedule<jupyter_text>Feature Extraction from training set* **BOW**<jupyter_code>@printExecTime
def bowBuilder(docs, max_features=None, min_df=1, max_df=1, binary=False):

    vectorizer = CountVectorizer(max_features=max_features, binary=binary)
    docs_bow = vectorizer.fit_transform(docs)
    return docs_bow, vectorizer

train_bow, vectorizer = bowBuilder(train_x, max_features=50000)
print(train_bow.shape)<jupyter_output>Execution time in sec for bowBuilder was 1.9526051999999936
(13179, 50000)<jupyter_text>Transform `test` to *BOW* using vocabulary from `train`  set<jupyter_code># dev_bow = vectorizer.transform(dev_x)
test_bow = vectorizer.transform(test_x)
# print("Shape dev", dev_bow.shape)
print("Shape test", test_bow.shape)
# print(train_bow.todense()[0].sum())
# vectorizer.get_feature_names()[30000:30200]
frequency_matrix = pd.DataFrame(train_bow.toarray(),
                                columns = vectorizer.get_feature_names())
frequency_matrix.iloc[0].sum()
# train_set[0][2]
# CLASSES[12]
# print('good' in vectorizer.get_feature_names())
# train_x[0]
# vectorizer.inverse_transform(train_bow)<jupyter_output><empty_output><jupyter_text>* **Binary BOW representation**<jupyter_code>train_bool, bool_vectorizer = bowBuilder(train_x,max_features=50000, binary=True)
# dev_bool = bool_vectorizer.transform(dev_x)
test_bool = bool_vectorizer.transform(test_x)

print("Shape bin train", train_bool.shape)
# print("Shape bin dev", dev_bool.shape)
print("Shape bin test", test_bool.shape)<jupyter_output>Execution time in sec for bowBuilder was 1.9789134000000104
Shape bin train (13179, 126329)
Shape bin test (5649, 126329)<jupyter_text>* **Tfidf representation**<jupyter_code>transformer = TfidfTransformer()
train_tfidf = transformer.fit_transform(train_bow)
test_tfidf = transformer.transform(test_bow)<jupyter_output><empty_output><jupyter_text>Training models on 70% training set  Logistic Regression.We will evaluate some multinomial logistic Regressions with different hyperparameters like L2 penalty and class weights<jupyter_code>def model(penalty=1, class_weight=None, solver='lbfgs'):
    classifier = LogisticRegression(multi_class='multinomial', solver=solver,
                                    C=penalty, class_weight=class_weight, n_jobs=6)
    return classifier

def saveModel(model, filename, folder='datasets'):
    path = os.path.join(os.getcwd(), folder, filename)
    with open(path, 'wb') as picklefile:
        pickle.dump(model, picklefile)

def readModel(filename, folder='datasets'):
    path = os.path.join(os.getcwd(), folder, filename)
    with open(path, 'rb') as model:
        return pickle.load(model)<jupyter_output><empty_output><jupyter_text>Saving model partitions for later use.<jupyter_code>saveModel(train_set, 'train_set')
saveModel(train_y, 'train_y')
saveModel(train_bow, 'train_bow')
saveModel(train_bool, 'train_bool')
saveModel(train_tfidf, 'train_tfidf')
saveModel(test_bow, 'test_bow')
saveModel(test_bool, 'test_bool')
saveModel(test_tfidf, 'test_tfidf')<jupyter_output><empty_output><jupyter_text>Definition of models/ hyper parameters combinations to evaluate for LR<jupyter_code>def getLRModels(solver='sag', repr='bow'):
    models = {}
    penalties = [0.01, 0.1, 0.5, 1]
    class_weights = ['balanced', None]
    for c in penalties:
        for w in class_weights:
            key = f'c-{c}-w-{w}-LR-{repr}'
            models[key] = model(penalty=c, class_weight=w, solver=solver)
    return models

def evaluate_model(model, X, y, metrics=['accuracy']):
    cv = KFold(n_splits=10)
    scores = cross_validate(model, X, y, scoring=metrics, cv=cv, n_jobs=-1)
    return scores<jupyter_output><empty_output><jupyter_text>Cross validate LR for BOW representation<jupyter_code>@printExecTime
def crossValidateModels(models, X, y, metrics=['accuracy']):
    results, names = [], []
    for name, model in models.items():
        scores = evaluate_model(model, X, y, metrics=metrics)
        results.append(scores)
        names.append(name)
        print('Got result for:', name)
    return results, names

def getPerClassScorings():
    recall = {f'recall-{i}': make_scorer(recall_score, average=None, labels=[i]) for i in range(1, 21)}
    precision = {f'precision-{i}': make_scorer(precision_score, average=None, labels=[i]) for i in range(1, 21)}
    scoring = recall.copy()
    scoring.update(precision)
    return scoring

bow_results, names_bow_LR = crossValidateModels(getLRModels(), train_bow, train_y, metrics=getPerClassScorings())
saveModel(bow_results, 'bow_results_LR')
bow_results = readModel('bow_results_LR')
print(names_bow_LR[0] ,len(bow_results[0]))<jupyter_output>c-0.01-w-balanced-LR-bow 42<jupyter_text>Cross validate LR binary BOW representation<jupyter_code>bow_bool_results, names_bool_LR = crossValidateModels(getLRModels(repr='bool'), train_bool, train_y, metrics=getPerClassScorings())
saveModel(bow_bool_results, 'bow_bool_results_LR')
bow_bool_results = readModel('bow_bool_results_LR')
print(names_bool_LR[0] ,len(bow_bool_results[0]))<jupyter_output>c-0.01-w-balanced-LR-bool 42<jupyter_text>Cross validate LR for tfidf representation<jupyter_code>tfidf_results_LR, names_tfidf_LR = crossValidateModels(getLRModels(repr='tfidf'), train_tfidf, train_y, metrics=getPerClassScorings())
saveModel(tfidf_results_LR, 'tfidf_results_LR')
tfidf_results_LR = readModel('tfidf_results_LR')
print(names_tfidf_LR[0] ,len(bow_bool_results[0]))<jupyter_output>c-0.01-w-balanced-LR-tfidf 42<jupyter_text>Naive BayesWe will repeat the above steps in these case using the Naive Bayes Multinomial Classifier implementation of`SKlearn`.The hyper parameters to evaluate will be:* alpha_smoothing ([0, 1])* fir_prior (boolean)<jupyter_code>def NBmodel(alpha, fit_prior):
    return MultinomialNB(alpha=alpha, fit_prior=fit_prior)

def getBayesModels(repr='bow'):
    smoothing = [0.01, 0.1, 0.5, 1]
    fit_prior = [True, False]
    models = {}
    for a in smoothing:
        for b in fit_prior:
            bin = str(b)[0]
            key = f'alpha-{a}-fp-{bin}-NB-{repr}'
            models[key] = NBmodel(alpha=a, fit_prior=b)
    return models<jupyter_output><empty_output><jupyter_text>Cross validate Naive Bayes for BOW representation<jupyter_code>bow_results_NB, names_bow_NB = crossValidateModels(getBayesModels(),
                                            train_bow, train_y, metrics=getPerClassScorings())<jupyter_output>Got result for: alpha-0.01-fp-T-NB-bow
Got result for: alpha-0.01-fp-F-NB-bow
Got result for: alpha-0.1-fp-T-NB-bow
Got result for: alpha-0.1-fp-F-NB-bow
Got result for: alpha-0.5-fp-T-NB-bow
Got result for: alpha-0.5-fp-F-NB-bow
Got result for: alpha-1-fp-T-NB-bow
Got result for: alpha-1-fp-F-NB-bow
Execution time in sec for crossValidateModels was 5.915110400000003<jupyter_text>Cross validate Naive Bayes for Binary BOW representation<jupyter_code>bow_boolNB_results, names_bool_NB = crossValidateModels(getBayesModels(repr='bool'),
                                                train_bool, train_y, metrics=getPerClassScorings())<jupyter_output>Got result for: alpha-0.01-fp-T-NB-bool
Got result for: alpha-0.01-fp-F-NB-bool
Got result for: alpha-0.1-fp-T-NB-bool
Got result for: alpha-0.1-fp-F-NB-bool
Got result for: alpha-0.5-fp-T-NB-bool
Got result for: alpha-0.5-fp-F-NB-bool
Got result for: alpha-1-fp-T-NB-bool
Got result for: alpha-1-fp-F-NB-bool
Execution time in sec for crossValidateModels was 7.419818300000088<jupyter_text>Cross validate Naive Bayes for TfIdf representation<jupyter_code>tfidf_NB_results, names_tfidf_NB = crossValidateModels(getBayesModels(repr='tfidf'),
                                                train_tfidf, train_y, metrics=getPerClassScorings())<jupyter_output>Got result for: alpha-0.01-fp-T-NB-tfidf
Got result for: alpha-0.01-fp-F-NB-tfidf
Got result for: alpha-0.1-fp-T-NB-tfidf
Got result for: alpha-0.1-fp-F-NB-tfidf
Got result for: alpha-0.5-fp-T-NB-tfidf
Got result for: alpha-0.5-fp-F-NB-tfidf
Got result for: alpha-1-fp-T-NB-tfidf
Got result for: alpha-1-fp-F-NB-tfidf
Execution time in sec for crossValidateModels was 5.817578700000013<jupyter_text>Cross Validation MetricsHere we will use a 10-fold cross validation strategy over the 70% `train` set to evaluate the bests hyperparametersfor our models from the previous chosen subset of all the possibles hyperparameters combinations<jupyter_code>def meanOfScores(scores, score_keys):
    avg_recalls, avg_precisions = [], []
    for key in score_keys:
        avgs = np.mean(scores['test_' + key])
        if 'recall' in key:
            avg_recalls.append(avgs)
        else:
            avg_precisions.append(avgs)
    return avg_recalls, avg_precisions


def modelsPerScore(scoresPerModels):
    all_recalls = []
    all_precisions = []

    for model_scores in scoresPerModels:
        avg_recalls, avg_precisions = meanOfScores(model_scores, getPerClassScorings().keys())
        all_recalls.append(avg_recalls)
        all_precisions.append(avg_precisions)

    return all_recalls, all_precisions


def graphMetrics(all_recalls, all_precisions, model_names):
    plt.rcParams["figure.figsize"] = (18,10)
    plt.subplot(2, 1, 1)
    plt.boxplot(all_recalls, showmeans=True, labels=model_names)
    plt.title('Boxplot of recalls distribution per class in various models')
    plt.xlabel('Models')
    plt.legend()
    plt.grid(0.2)

    plt.subplot(2, 1, 2)
    plt.boxplot(all_precisions, showmeans=True, labels=model_names)
    plt.title('Boxplot of precision distribution per class in various models')
    plt.xlabel('Models')
    plt.legend()
    plt.grid(0.2)
    plt.tight_layout()
    plt.show()<jupyter_output><empty_output><jupyter_text>Here we get for each model, average recall and precision per class. This is used to graph a box plot from the metricsdistributions for the classes in each model.**Note:** The model names correspond to the type of model, its hyperparameters, and the document representation beeingused for the cross validation.<jupyter_code>bow_recalls, bow_precisions = modelsPerScore(bow_results)
bow_bool_recalls, bow_bool_precisions = modelsPerScore(bow_bool_results)
tfidf_recalls, tfidf_precisions = modelsPerScore(tfidf_results_LR)<jupyter_output><empty_output><jupyter_text>Graph Logistic Regression* **BOW - Logistic Regression**. Distribution of class metrics in `train` set<jupyter_code>graphMetrics(bow_recalls, bow_precisions, names_bow_LR)<jupyter_output>No handles with labels found to put in legend.
No handles with labels found to put in legend.<jupyter_text>* **Binary BOW - Logistic Regression**. Distribution of class metrics in `train` set<jupyter_code>graphMetrics(bow_bool_recalls, bow_bool_precisions, names_bool_LR)<jupyter_output>No handles with labels found to put in legend.
No handles with labels found to put in legend.<jupyter_text>* **TfIdf - Logistic Regression**. Distribution of class metrics in `train` set<jupyter_code>graphMetrics(tfidf_recalls, tfidf_precisions, names_tfidf_LR)<jupyter_output>No handles with labels found to put in legend.
No handles with labels found to put in legend.<jupyter_text>Graph Naive Bayes<jupyter_code>bow_recalls_NB, bow_precisions_NB = modelsPerScore(bow_results_NB)
bow_bool_recalls_NB, bow_bool_precisions_NB = modelsPerScore(bow_boolNB_results)
tfidf_recalls_NB, tfidf_precisions_NB = modelsPerScore(tfidf_NB_results)<jupyter_output><empty_output><jupyter_text>* **BOW - Naive Bayes**. Distribution of class metrics in `train` set<jupyter_code>graphMetrics(bow_recalls_NB, bow_precisions_NB, names_bow_NB)<jupyter_output>No handles with labels found to put in legend.
No handles with labels found to put in legend.<jupyter_text>* **Binary BOW - Naive Bayes**. Distribution of class metrics in `train` set<jupyter_code>graphMetrics(bow_bool_recalls_NB, bow_bool_precisions_NB, names_bool_NB)<jupyter_output>No handles with labels found to put in legend.
No handles with labels found to put in legend.<jupyter_text>* **TfIdf - Naive Bayes**. Distribution of class metrics in `train` set<jupyter_code>graphMetrics(tfidf_recalls_NB, tfidf_precisions_NB, names_tfidf_NB)<jupyter_output>No handles with labels found to put in legend.
No handles with labels found to put in legend.<jupyter_text>Escogencia de hiperparámetros.**LR**En este caso la representación que convergió significativamente más rápido fue *tfidf*. Esto en parte por que son vectores mucho menos dispersos y de magnitudes similares.Por ende, escogeremos los hiperparámetros de LR que mayor hayan rendido en el caso *tfid*. un *c penalty* de 1 y *weighted classed* en `None`.**NB**En Naive Bayes, todos los modelos fueron rápidos en comparación con los de **LR**. Por ende escogeremos los hipeerparámetros que se comportenaproximadamente mejor que el resto en las 6 gráficas. Esto corresponde a un *alpha smoothing* cercano a 0 con fit prior `True`TODO<jupyter_code>def getFinalModels():
    c, cw = 1, None
    alpha, fit_prior = 0, True
    LR_model, NB_model = model(penalty=c, class_weight=cw), NBmodel(alpha=alpha, fit_prior=fit_prior)
    LR_models = LR_model, copy.copy(LR_model), copy.copy(LR_model)
    NB_models = NB_model, copy.copy(NB_model), copy.copy(NB_model)
    return LR_models, NB_models

[LR_bow_model, LR_bool_model, LR_tfidf_model],\
[NB_bow_model, NB_bool_model, NB_tfidf_model] = getFinalModels()<jupyter_output><empty_output><jupyter_text>Train final models for each Representation<jupyter_code>def trainFinalModels(LR_models, NB_models, y_train, *repr):
    assert(len(LR_models)==len([*repr]))
    assert(len(NB_models)==len([*repr]))
    for i in range(len([*repr])):
        LR_models[i].fit([*repr][i], y_train)
        print(f'LR model #{i+1} fitted')
        NB_models[i].fit([*repr][i], y_train)
        print(f'NB model #{i+1} fitted')
trainFinalModels([LR_bow_model, LR_bool_model, LR_tfidf_model],
                [NB_bow_model, NB_bool_model, NB_tfidf_model],
                 train_y,
                train_bow, train_bool, train_tfidf)<jupyter_output>LR model #1 fitted
NB model #1 fitted
LR model #2 fitted
NB model #2 fitted
LR model #3 fitted
NB model #3 fitted<jupyter_text>Save trained models<jupyter_code>saveModel(LR_bow_model, 'final_LR_bow')
saveModel(LR_bool_model, 'final_LR_bool')
saveModel(LR_tfidf_model, 'final_LR_tfidf')
saveModel(NB_bow_model, 'final_NB_bow')
saveModel(NB_bool_model, 'final_NB_bool')
saveModel(NB_tfidf_model, 'final_NB_tfidf')
LR_bow_model = readModel('final_LR_bow')
LR_bool_model = readModel('final_LR_bool')
LR_tfidf_model = readModel( 'final_LR_tfidf')
NB_bow_model = readModel('final_NB_bow')
NB_bool_model = readModel('final_NB_bool')
NB_tfidf_model = readModel('final_NB_tfidf')
train_bow = readModel( 'train_bow')
train_bool = readModel( 'train_bool')
train_tfidf = readModel( 'train_tfidf')
test_bow = readModel( 'test_bow')
test_bool = readModel( 'test_bool')
test_tfidf = readModel( 'test_tfidf')
final_models = [LR_bow_model, LR_bool_model, LR_tfidf_model, NB_bow_model, NB_bool_model, NB_tfidf_model]
final_names = ['final_LR_bow', 'final_LR_bool', 'final_LR_tfidf',
               'final_NB_bow', 'final_NB_bool', 'final_NB_tfidf']<jupyter_output><empty_output><jupyter_text>Evaluate final classifiers with test set<jupyter_code>scoring = {'f1_micro': partial(f1_score, average='micro'),
              'fi_macro': partial(f1_score, average='macro'),
              'recall_micro': partial(recall_score, average='micro'),
              'recall_macro': partial(recall_score, average='macro'),
              'precision_micro': partial(precision_score, average='micro'),
              'precision_macro': partial(precision_score, average='macro'),
              'accuracy': accuracy_score}

def evaluateClassifier(model, X_test , y, scoring):

    y_pred = model.predict(X_test)
    model_scores = {key: value(y_true=y, y_pred=y_pred) for key, value in scoring.items()}

    return model_scores

def evaluateAllModels(models, names, X_test, y_test,  scoring):
    scores = {}
    for i in range(len(models)):
        scores[names[i]] = evaluateClassifier(models[i],
                                              X_test[i],
                                              y_test, scoring)
    return scores
scores = evaluateAllModels(final_models, final_names,
                           [test_bow, test_bool, test_tfidf,
                            test_bow, test_bool, test_tfidf],
                           test_y,
                           scoring)
print(scores)<jupyter_output>{'final_LR_bow': {'f1_micro': 0.8929013984776066, 'fi_macro': 0.8917233967802772, 'recall_micro': 0.8929013984776066, 'recall_macro': 0.8909877804699166, 'precision_micro': 0.8929013984776066, 'precision_macro': 0.8936189469209742, 'accuracy': 0.8929013984776066}, 'final_LR_bool': {'f1_micro': 0.8948486457780138, 'fi_macro': 0.8935474769087918, 'recall_micro': 0.8948486457780138, 'recall_macro': 0.8925127455312827, 'precision_micro': 0.8948486457780138, 'precision_macro': 0.8956173631924585, 'accuracy': 0.8948486457780138}, 'final_LR_tfidf': {'f1_micro': 0.9063551070986016, 'fi_macro': 0.9037563690212075, 'recall_micro': 0.9063551070986016, 'recall_macro': 0.9025522106026962, 'precision_micro': 0.9063551070986016, 'precision_macro': 0.9073632956674113, 'accuracy': 0.9063551070986016}, 'final_NB_bow': {'f1_micro': 0.8766153301469286, 'fi_macro': 0.8739217162377952, 'recall_micro': 0.8766153301469286, 'recall_macro': 0.874421104551104, 'precision_micro': 0.8766153301469286, 'precision_ma[...]<jupyter_text>Pretty print results<jupyter_code>score_names = ['f1_micro', 'fi_macro', 'recall_micro',
               'recall_macro','precision_micro','precision_macro','accuracy']

table = [value for value in scores.values()]
results = pd.DataFrame(table, index=final_names)
results<jupyter_output><empty_output><jupyter_text>LR vs NB<jupyter_code>LR_results = results.iloc[:3].mean(axis=0).to_list()
NB_results = results.iloc[3:].mean(axis=0).to_list()
LRvsNB = pd.DataFrame.from_records([LR_results, NB_results], columns=score_names, index=['LR', 'NB'])
LRvsNB<jupyter_output><empty_output><jupyter_text>Bow vs Binary Bow vs Tfidf<jupyter_code>BOW_results = results.iloc[[0, 3]].mean(axis=0).to_list()
Bool_results = results.iloc[[1, 4]].mean(axis=0).to_list()
Tfidf_results = results.iloc[[2, 5]].mean(axis=0).to_list()

BowvsBoolvsTfidf = pd.DataFrame.from_records([BOW_results, Bool_results, Tfidf_results], columns=score_names, index=['Bow', 'Binary Bow', 'Tfidf'])
BowvsBoolvsTfidf<jupyter_output><empty_output><jupyter_text>Per Class metrics of the whole dataset<jupyter_code>bow_all = vstack((train_bow, test_bow))
bool_all = vstack((train_bool, test_bool))
tfidf_all = vstack((train_tfidf, test_tfidf))
y_all = train_y + test_y
print('Shape of all' , train_bow.shape)
print('Shape of test' , test_bow.shape)
print('Shape of train' , bow_all.shape)
print('Shape of results', len(y_all))
def getScoringTotal():

    recall = {f'recall-{i}': partial(recall_score, average=None, labels=[i]) for i in range(1, 21)}
    precision = {f'precision-{i}': partial(precision_score, average=None, labels=[i]) for i in range(1, 21)}
    f1 = {f'f1-{i}': partial(f1_score, average=None, labels=[i]) for i in range(1, 21)}
    return {**recall, **precision, **f1}
total_scores = scores = evaluateAllModels(final_models, final_names,
                           [bow_all, bool_all, tfidf_all,
                            bow_all, bool_all, tfidf_all],
                           y_all,
                           getScoringTotal())
def prettyPrintScores(total_scores):
    table = [
        {'recall': [], 'precision': [], 'f1': []}
        for i in range(20)]

    for model_scores in total_scores.values():
        for key, score in model_scores.items():
            class_index = int(key.split('-')[1])
            score_name = key.split('-')[0]
            table[class_index-1][score_name].append(score[0])

    table = [{ key: sum(value)/len(value)
               for key, value in class_scores.items()
               }for class_scores in table]
    return table
# total_scores
table = prettyPrintScores(total_scores)
class_names = [CLASSES[i+1] for i in range(20)]
score_per_class = pd.DataFrame(table, index=class_names)
score_per_class<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11432825294832112, 'fraction_numerical': 0.03181444187595502, 'mean_word_length': 4.93443385259242, 'pattern_counts': {'":': 0, '<': 121, '<?xml version=': 0, '>': 121, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '35', 'dir': 'jupyter-structured-clean-dedup', 'id': '629330', 'license': 'MIT', 'n_tokens_mistral': 9665, 'n_tokens_neox': 8777, 'n_words': 1918, 'path': 'HW2/NB_LR.ipynb', 'repo_name': 'diegommezp28/NLP_202110', 'size': '25523'}