<jupyter_start><jupyter_text>Task 1This Task consists of using the BBC Dataset to teach a Multinomial Naive Bayes classifier to distinguish between different article types. A given article should be classified into 1 of 5 different categories: business,entertainment, politics, sport, and tech<jupyter_code># Import Dependencies
import numpy as np
import matplotlib.pyplot as plt
import os, os.path
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
import sklearn.datasets as datasets
from pathlib import Path
# Import Dataset
from google.colab import drive
# drive.mount('/content/drive', force_remount=True) # Only for development
!gdown --id 1hg8v4l7iGcqW83tNwbBRlVaFtfNCn6rr
!unzip /content/BBC.zip
# Plot distribution of BBC Dataset

## Set up data + labels
dir = "/content/BBC"
y = np.array([])
curr_label = 1
labels = ["Business", "Entertainment", "Politics", "Sports", "Tech"]

## create probability ditribution by adding a particular label to an array n times
## where n is the the number of files per folder
for name in os.listdir(dir):
  if os.path.isdir(os.path.join(dir, name)) and not name.startswith("."):
    y = np.append(y, np.repeat(curr_label, len(os.listdir(os.path.join(dir, name)))))
    curr_label += 1
    
## Set up the plot
plt.xticks(np.arange(1, 6), labels)
n, bins, patches = plt.hist(y, bins=5, facecolor='b', alpha=0.75, ec="k")
plt.show()
# Load Corpus
corpus = datasets.load_files(dir, encoding='latin1')
y = corpus.target
# Pre-Process Data set

## vectorize texts into word frequencies for model use
vectorizer = CountVectorizer()
X_text = vectorizer.fit_transform(corpus['data'])
print(X_text.shape, y.shape)
# Split Dataset
X_train, X_test, y_train, y_test = train_test_split(X_text, y, test_size=.20, random_state=None)<jupyter_output><empty_output><jupyter_text>---Naive Bayes: Try 1---<jupyter_code># Train Naive Bayes: Try 1
nb_one = MultinomialNB()
nb_one.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>(B)Confusion Matrix<jupyter_code>print (nb_one.score(X_test, y_test))
y_pred = nb_one.predict(X_test)
# print(y_test)
# print(y_pred)
print(confusion_matrix(y_test, y_pred))<jupyter_output>0.9775280898876404
[[ 96   0   2   0   3]
 [  0  74   1   0   1]
 [  1   0  91   0   1]
 [  0   0   0 100   0]
 [  0   1   0   0  74]]<jupyter_text>(C)Precision, Recall, F1-measure<jupyter_code>print(classification_report(y_test, y_pred, target_names=labels))<jupyter_output>precision    recall  f1-score   support

     Business       0.99      0.95      0.97       101
Entertainment       0.99      0.97      0.98        76
     Politics       0.97      0.98      0.97        93
       Sports       1.00      1.00      1.00       100
         Tech       0.94      0.99      0.96        75

     accuracy                           0.98       445
    macro avg       0.98      0.98      0.98       445
 weighted avg       0.98      0.98      0.98       445<jupyter_text>(D) Accuracy<jupyter_code>print(accuracy_score(y_test, y_pred))
print(accuracy_score(y_test, y_pred, normalize=False))<jupyter_output>0.9775280898876404
435<jupyter_text>Macro average F1<jupyter_code>f1_score(y_test, y_pred, average = 'macro')<jupyter_output><empty_output><jupyter_text>Weighted average F1<jupyter_code>f1_score(y_test, y_pred, average = 'weighted')<jupyter_output><empty_output><jupyter_text>(E)Prior probability of each class<jupyter_code># nb.Proba()???<jupyter_output><empty_output><jupyter_text>(F) Size of Vocabulary<jupyter_code>len(vectorizer.vocabulary_)<jupyter_output><empty_output><jupyter_text>(E) Number of Word-tokens in each class<jupyter_code>class_count = [0, 0, 0, 0, 0]
array_text = X_text.toarray()

for index in range(len(array_text)):
  class_count[y[index]-1] += sum(array_text[index])

print(class_count)<jupyter_output>[124893, 185208, 162953, 198640, 164663]<jupyter_text>(H) Number of Word-tokens in corpus<jupyter_code>sum(map(sum, array_text))<jupyter_output><empty_output><jupyter_text>(I) Words with Freq of zero in each class<jupyter_code>class_non_zero_count = [0, 0, 0, 0, 0]
# ahhh<jupyter_output><empty_output><jupyter_text>(I) Words with Freq of one in entire corpus<jupyter_code><jupyter_output><empty_output><jupyter_text>Naive Bayes: Try 2Note: cells displaying model performance will be ommited in all future tries in favor of generating an addition to the text file report.<jupyter_code># Train Naive Bayes: Try 2
nb_two = MultinomialNB()
nb_two.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Naive Bayes: Try 3<jupyter_code># Train Naive Bayes: Try 3
nb_three = MultinomialNB(alpha=0.0001)
nb_three.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Naive Bayes: Try 4<jupyter_code># Train Naive Bayes: Try 4
nb_four = MultinomialNB(alpha=0.9)
nb_four.fit(X_train, y_train)
# Generate / Appened to Performance file 

# model is the instance of NB, Label is the text that will show at the top of the run
# 
def generate_report(model, label, vector, dataset):
  """
  Appends to a report file with the statistics of a particular model run.

  Parameters
  ----------
  model : object
    Instance of the machine learning model, NB, Decision tree, etc
  label : string
    String that will appear at the top of a particular run in the text file
  vector : object
    Count vectorizor class, fit on the current corpus
  dataset : object
    the fit_transform output of the above vectorizor

  """
  f = open("bbc_performance.txt", "a")
  f.write("\n==========================================================")
  f.write("\nAttempt Description: " + label)

  f.write("\nConfusion Martix + Score:\n")
  f.write(str(model.score(X_test, y_test)))
  f.write("\n")
  y_pred = model.predict(X_test)
  f.write(str(confusion_matrix(y_test, y_pred)))

  f.write("\nClassification Report (precision, recall, and F1-measure )\n")
  f.write(str(classification_report(y_test, y_pred, target_names=labels)))

  f.write("\nAccuracy Score: accuracy, macro-average F1 and weighted-average F1\n")
  f.write(str(accuracy_score(y_test, y_pred, normalize=False)))
  f.write("\n")
  f.write(str(f1_score(y_test, y_pred, average = 'macro')))
  f.write("\n")
  f.write(str(f1_score(y_test, y_pred, average = 'weighted')))
  f.write("\n")

  f.write("\nSize of Vocabulary\n")
  f.write(str(len(vector.vocabulary_)))

  f.write("\nNumber of Word Tokens in each class\n\n")
  class_count = [0, 0, 0, 0, 0]
  array_text = dataset.toarray()

  for index in range(len(X_text.toarray())):
    class_count[y[index]-1] += sum(array_text[index])

  f.write(str(class_count))
  
  f.write("\n")
  
  f.write("\nNumber of Word Tokens in Entire Corpus\n\n")
  f.write(str(sum(map(sum, dataset.toarray()))))

  f.close()



generate_report(nb_one, "Multi-nomialNB default values, try 1", vectorizer, X_text)
generate_report(nb_two, "Multi-nomialNB default values, try 2", vectorizer, X_text)
generate_report(nb_three, "Multi-nomialNB with 0.0001 smoothing, try 3", vectorizer, X_text)
generate_report(nb_four, "Multi-nomialNB with 0.9 smoothing, try 4", vectorizer, X_text)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12269021739130435, 'fraction_numerical': 0.035461956521739134, 'mean_word_length': 4.662307692307692, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '509018', 'license': 'MIT', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2614, 'n_words': 653, 'path': 'Project 1/Project1_task1.ipynb', 'repo_name': 'm-triassi/ai-projects-472', 'size': '7360'}