<jupyter_start><jupyter_text>W207 Group Project: True vs Fake News  Members: <NAME>, <NAME>, <NAME>, <NAME>  Project BackgroundOur goal for this project was to use labeled news datasets to make an algorithm that is able to determine if an article is fake news or real news. To build the model, we used a dataset provided by Kaggle. During the model development process, we explored performance of Logistic, Naive Bayes, and KNN models, along with unigram, bigram, and trigram feature vectors for text representation. To test the generalizability of the models, we then evaluated the models against a second dataset provided by Kaggle.  Data Background Dataset 1* Provided by Kaggle: https://www.kaggle.com/clmentbisaillon/fake-and-real-news-dataset* Contents: two data sets -- one that includes labeled fake news articles and one that includes labeled real news articles* Primary focus of our analysis Dataset 2* Provided by Kaggle: https://www.kaggle.com/c/fake-news/code* Contents: two data sets -- one that includes labeled train data and one that includes unlabeled test data* Supplementary dataset to explore generalizability of the models<jupyter_code># Load libraries.
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import zipfile

import time
import re
import string
import nltk
import unicodedata

from nltk.corpus import stopwords
from textblob import TextBlob

from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.feature_extraction import text
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, accuracy_score
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Import Dataset 1<jupyter_code># Read in fake news data file.
fake_zip = zipfile.ZipFile("input/fake-and-real-news-dataset/Fake.csv.zip") 
fake = pd.read_csv(fake_zip.open("Fake.csv"))
fake.head()
# Read in true news data file.
true_zip = zipfile.ZipFile("input/fake-and-real-news-dataset/True.csv.zip") 
true = pd.read_csv(true_zip.open("True.csv"))
true.head()
# Create label columns for the fake news data set.
fake["label"] = "false"
fake["label_bin"] = 0
fake.shape
# Print the count of fake records by subject. 
fake['subject'].value_counts()
# Create label columns for the true news data set.
true["label"] = "true"
true["label_bin"] = 1
true.shape
# Print the count of true records by subject.
true['subject'].value_counts()
# Merge + combine data sets into one.
data = [fake, true]
df = pd.concat(data, ignore_index=True)
df.shape<jupyter_output><empty_output><jupyter_text>Clean & Process Dataset 1<jupyter_code>df.info()
# Confirm the population of each column.
print(f"Null records:\n{df.isna().sum()}\n")
print(f"Empty text records: {len(df[df['text'] == ' '])}\n")
print(f"Empty title records: {len(df[df['title'] == ''])}")
# Find & remove records where the cleaned text or title column values are empty or NULL.
print(f"Number of text records with a URL: {df['text'].str.count('http|www').sum()}")
print(f"Number of title records with a URL: {df['title'].str.count('http|www').sum()}")
print(f"Number of date records with a URL: {df['date'].str.count('http|www').sum()}")
# Find records where date column values are invalid.
df["date"] = pd.to_datetime(df["date"], errors="coerce").dt.date
print(f"Number of invalid date records: {len(df.loc[df['date'].isnull()])}")
df.loc[df['date'].isnull()].head(20)
# Remove records where date column values are invalid.
df = df.dropna(subset=["date"])
df.shape
# Explore the date column.
df["date"].describe()
# Combine the title and text columns into one column.
df['titletext'] = df['title'] + ' ' + df['text']
df.head()
# Check for special characters in the the title and text columns that look like social posts.
print(f"Number of times titletext records contain '@' or '#': {df['text'].str.count('#|@').sum()}")
df[df["titletext"].str.contains("#|@")]
# Function for cleaning text inputs.
def preprocess_text(text_1):
    """Use regex and other tools for pre-processing of the text."""
    # Apply lowercasing.
    text = text_1.lower()
    
    # Replace stop words with empty strings.
    pattern = re.compile(r"\b(" + r"|".join(stopwords.words('english')) + r')\b\s*')
    text = pattern.sub("", text)
    
    # Replace punctuation with empty strings.
    text = re.sub(f"[{re.escape(string.punctuation)}]", "", text)
    
    # Replace all non-alphanumeric characters with a single space.
    text = re.sub(r"[^a-z0-9]+", " ", text)
    
    # Replace large, consecutive amounts of whitespace with a single space.
    text = re.sub(r"[\s]+", " ", text)
    
    # Remove leading and trailing whitespace.
    text = text.strip()
    
    return text
# Clean up text features.
nltk.download("stopwords")

start_time = time.time()
stop_words = set(stopwords.words("english"))

df["clean_title"] = df["title"].apply(lambda text: preprocess_text(text))
df["clean_text"] = df["text"].apply(lambda text: preprocess_text(text))
df["clean_titletext"] = df["titletext"].apply(lambda text: preprocess_text(text))

print(f"{(time.time() - start_time):0.3f} seconds to execute pre-processing text inputs...")
# Remove single character words.
pd.options.display.max_colwidth = 500
def singlecharacter(text):
    final_text = []
    words = text.split()
    for i in words:
        if len(i) > 1:
            final_text.append(i)
    return " ".join(final_text)

df['clean_text'] = df['clean_text'].apply(singlecharacter)
# View the cleaned dataset.
df.head()
df["clean_text"].describe()
df["clean_title"].describe()
# Get count of empty clean text records.
print(f"Empty clean text records: {len(df[df['clean_text'] == ''])}")
df[df["clean_text"] == ""].head()
# Remove empty clean text records.
df = df[df["clean_text"] != ""]

# Confirm count of empty clean text records.
print(f"Empty clean text records (after removal): {len(df[df['clean_text'] == ''])}")
print(f"Filtered df size: {df.shape}")
df[df["clean_text"] == ""].head()
# Get count of empty clean title records.
print(f"Empty clean title records: {len(df[df['clean_title'] == ''])}")
df[df["clean_title"] == ""].head()
# Get count of special characters in the cleaned text column that look like social posts.
print(f"Number of times clean_text records contain '@' or '#': {df['clean_text'].str.count('#|@').sum()}")
# Create a word cloud of all the true records.
truefilt = df[(df['label_bin'] == 1)]
text = " ".join(article for article in truefilt.clean_text)

# print the number of words across all the articles
print ("There are {} words across all the true articles.".format(len(text)))

# Create and generate a word cloud image
wordcloud = WordCloud(max_words=100, background_color="white").generate(text)

# Display the wordcloud
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()
# Create a word cloud of all the fake records.
fakefilt = df[(df['label_bin'] == 0)]
text = " ".join(article for article in fakefilt.clean_text)

# print the number of words across all the articles
print ("There are {} words across all the fake articles.".format(len(text)))

# Create and generate a word cloud image
wordcloud = WordCloud(max_words=100, background_color="white").generate(text)

# Display the wordcloud
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output>There are 40978019 words across all the articles.<jupyter_text>As these wordclouds show, there are many similarities between the most popular words in the fake news articles and the most popular words in the real news articles. Further, the number of unique words is greater across the fake news articles than it is across the real news articles.  Split Train, Test, & Dev Data for Dataset 1<jupyter_code># Split cleaned data set into train (70%), test (15%), & dev (15%).
# Use random_state param to reproduce shuffle outcomes.
train, test = train_test_split(df, random_state=0, train_size=0.7, test_size=0.3, shuffle=True)
num_test = int(len(test)/2)
test = test[num_test:]
dev = test[:num_test]

print("Train data shape:", train.shape)
print("Test data shape:", test.shape)
print("Dev data shape:", dev.shape)
test.columns
# Export train, test, & dev data sets to dir.
# Write a Pandas DataFrame as a zipped CSV file.
# train.to_csv("input/processed-datasets/train.csv", index=False)
# test.to_csv("input/processed-datasets/test.csv", index=False)
# dev.to_csv("input/processed-datasets/dev.csv", index=False)<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis for Dataset 1<jupyter_code># Histograms looking at the distribution of fake vs real news labels.
g = sns.catplot(data=df, x="label", kind="count")
g.set(xlabel="Label", ylabel="# of Records", title="Fake vs Real News Frequency")
g.set_xticklabels(rotation=45)
# Histogram looking at the frequency of fake news subjects.
g = sns.catplot(data=fake, x="subject", col="label_bin", kind="count", aspect=2)
g.set(xlabel="News Subject", ylabel="Fake News Count", title="Subject Frequency: Fake News")
g.set_xticklabels(rotation=45)
# Histogram looking at the frequency of true news subjects.
g = sns.catplot(data=true, x="subject", col="label_bin", kind="count", aspect=2)
g.set(xlabel="News Subject", ylabel="True News Count", title="Subject Frequency: True News")
g.set_xticklabels(rotation=45)<jupyter_output><empty_output><jupyter_text>Though there is no overlap in subjects between fake news and real news, we don't see this as something that will pose a concern since we are not including the subject of the article in the model.<jupyter_code># Distribution of the word count of the clean text values of the news data.
df["word_count"] = df["clean_text"].apply(lambda x: len(str(x).split()))
sns.displot(df["word_count"], kde=True)
fig = plt.gcf()
# Distribution of the polarity of the clean text values of the news data. 
# Appears to be neutral on average.
df["polarity"] = df["clean_text"].map(lambda text: TextBlob(text).sentiment.polarity)
sns.displot(df["polarity"], kde=True)
fig = plt.gcf()
# Histogram of word length in fake news vs. in real news.
fig,(ax1,ax2) = plt.subplots(1, 2, figsize = (12, 8))
ylim = [0, 12000]
ax1.set_ylim(ylim)
ax2.set_ylim(ylim)
xlim = [0, 4000]
ax1.set_xlim(xlim)
ax2.set_xlim(xlim)
text_length = df[df['label_bin'] == 1]['clean_text'].str.split().map(lambda x: len(x))
print("Word count in longest real article:", max(text_length))
print("Average word count in real news articles:", sum(text_length)/len(text_length))
ax1.hist(text_length, color='green', bins = 25)
ax1.set_title('Real Articles')
text_length = df[df['label_bin'] == 0]['clean_text'].str.split().map(lambda x: len(x))
print("Word count in longest fake article:", max(text_length))
print("Average word count in fake news articles:", sum(text_length)/len(text_length))
ax2.hist(text_length, color = 'blue', bins = 25)
ax2.set_title('Fake Articles')
fig.suptitle('Number of Words per Article')
plt.show()<jupyter_output>Word count in longest real article: 2392
Average word count in real news articles: 228.17594322002242
Word count in longest fake article: 4830
Average word count in fake news articles: 237.142119089317<jupyter_text>The above histograms and calculations reveal that fake news articles are on average longer than real news articles, and the longest fake news article is over twice as long as the longest real news article.<jupyter_code># Histogram of character count in fake news vs. in real news.
fig,(ax1,ax2) = plt.subplots(1, 2, figsize = (12, 8))
ylim = [0, 15000]
ax1.set_ylim(ylim)
ax2.set_ylim(ylim)
xlim = [0, 20000]
ax1.set_xlim(xlim)
ax2.set_xlim(xlim)
text_length = df[df['label_bin'] == 1]['clean_text'].str.len()
print("Character count in longest real article:", max(text_length))
ax1.hist(text_length, color='green', bins = 25)
ax1.set_title('Real Articles')
text_length = df[df['label_bin'] == 0]['clean_text'].str.len()
print("Character count in longest fake article:", max(text_length))
ax2.hist(text_length, color = 'blue', bins = 25)
ax2.set_title('Fake Articles')
fig.suptitle('Number of Characters per Article')
plt.show()<jupyter_output>Character count in longest real article: 17958
Character count in longest fake article: 38702<jupyter_text>Similarly, the fake news article with the highest character count has over twice as many characters as the real news article with the highest character count.  Word CountsGet the Top 10 words for real and fake news datasets<jupyter_code>df_word_cnt_true = df[df['label']=='true']['clean_text'].str.split(expand=True).stack().value_counts().reset_index()
df_word_cnt_true.columns = ['word', 'freq']
df_word_cnt_true.head(10)
df_word_cnt_false = df[df['label']=='false']['clean_text'].str.split(expand=True).stack().value_counts().reset_index()
df_word_cnt_false.columns = ['word', 'freq']
df_word_cnt_false.head(10)<jupyter_output><empty_output><jupyter_text>In looking at the top 10 words for fake and real datasets, we see that "trump" and "said" are the top to words for both. When we saw that "reuters" was the third most common word in real news datasets, and since this word also appeared in the word cloud for real news datasets, we decided to explore this further. In looking more closely at the text data for the real news dataset, we saw that many of the entries start with the publisher in the text string, so we decided to add both "washington" and "reuters" to the list of stop words so that the models are not overfit to the training dataset.  Import Dataset 2<jupyter_code># Read in test dataset. Test dataset does not have labels.
test_zip = zipfile.ZipFile("./input/fake-news-dataset2/test.csv.zip") 
test = pd.read_csv(test_zip.open("test.csv"))
test.head()
# Read in Train dataset. Labels: 1: unreliable ; 0: reliable.
train_zip = zipfile.ZipFile("./input/fake-news-dataset2/train.csv.zip") 
df_test2 = pd.read_csv(train_zip.open("train.csv"))
df_test2.head()
# Update labels to match original dataset. 
df_test2['label_bin'] = df_test2['label'].map({1:0 , 0:1})
df_test2['label'] = df_test2['label'].map({1:'false' , 0:'true'})
df_test2.head()
df_test2.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 20800 entries, 0 to 20799
Data columns (total 6 columns):
 #   Column     Non-Null Count  Dtype 
---  ------     --------------  ----- 
 0   id         20800 non-null  int64 
 1   title      20242 non-null  object
 2   author     18843 non-null  object
 3   text       20761 non-null  object
 4   label      20800 non-null  object
 5   label_bin  20800 non-null  int64 
dtypes: int64(2), object(4)
memory usage: 975.1+ KB<jupyter_text>Clean & Process Dataset 2<jupyter_code># Check for empty records.
df_test2.isna().sum().sum()
# Check columns labeled as the 'object' dtype.
df_test2["title"].apply(type).value_counts()
# Check columns labeled as the 'object' dtype.
df_test2["text"].apply(type).value_counts()
# Inspect float objects in title column.
df_test2_inspect_nulls = df_test2[df_test2["title"].isna()]
df_test2_inspect_nulls.head()
# Inspect float objects in text column.
df_test2_inspect_nulls = df_test2[df_test2["text"].isna()]
df_test2_inspect_nulls.head()
# Remove NaN records from the text and title columns.
df_test2_filtered = df_test2.dropna(subset=["title", "text"])

# Confirm count of empty clean text records.
print(f"NaN text records (after removal): {len(df_test2_filtered[df_test2_filtered['text'] == ''])}")
print(f"NaN title records (after removal): {len(df_test2_filtered[df_test2_filtered['title'] == ''])}")
print(f"Filtered df size: {df_test2_filtered.shape}\n")
df_test2_filtered.info()
# Combine the title and text columns into one column.
df_test2_filtered['titletext'] = df_test2_filtered['title'] + ' ' + df_test2_filtered['text']

# Clean up text features.
start_time = time.time()
stop_words = set(stopwords.words("english"))

df_test2_filtered["clean_title"] = df_test2_filtered["title"].apply(lambda text: preprocess_text(text))
df_test2_filtered["clean_text"] = df_test2_filtered["text"].apply(lambda text: preprocess_text(text))
df_test2_filtered["clean_titletext"] = df_test2_filtered["titletext"].apply(lambda text: preprocess_text(text))

print(f"{(time.time() - start_time):0.3f} seconds to execute pre-processing text inputs...")
df_test2_filtered['clean_text'] = df_test2_filtered['clean_text'].apply(singlecharacter)
df_test2_filtered.head(2)
# Get count of empty clean text records.
print(f"Empty clean text records: {len(df_test2_filtered[df_test2_filtered['clean_text'] == ''])}")
df_test2_filtered[df_test2_filtered["clean_text"] == ""].head()
# Remove empty clean text records.
df_test2_filtered = df_test2_filtered[df_test2_filtered["clean_text"] != ""]

# Confirm count of empty clean text records.
print(f"Empty clean text records (after removal): {len(df_test2_filtered[df_test2_filtered['clean_text'] == ''])}")
print(f"Filtered df size: {df_test2_filtered.shape}")
df_test2_filtered[df_test2_filtered["clean_text"] == ""].head()
# Get count of empty clean title records.
print(f"Empty clean title records: {len(df_test2_filtered[df_test2_filtered['clean_title'] == ''])}")
df_test2_filtered[df_test2_filtered["clean_title"] == ""].head()
# Remove empty clean title records.
df_test2_filtered = df_test2_filtered[df_test2_filtered["clean_title"] != ""]

# Confirm count of empty clean title records.
print(f"Empty clean text records (after removal): {len(df_test2_filtered[df_test2_filtered['clean_title'] == ''])}")
print(f"Filtered df size: {df_test2_filtered.shape}")
df_test2_filtered[df_test2_filtered["clean_title"] == ""].head()
# Get count of special characters in the the title and text columns that look like social posts.
print(f"Number of times clean_text records contain '@' or '#': {df_test2_filtered['clean_text'].str.count('#|@').sum()}")
df_test2_filtered.shape<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis for Dataset 2<jupyter_code># See the counts of true and fake news articles.
pd.pivot_table(df_test2_filtered, values="id", index="label", aggfunc="count")
# Create a word cloud of all the true records.
truefilt = df_test2_filtered[(df_test2_filtered['label_bin'] == 1)]
text = " ".join(article for article in truefilt.clean_text)

# Print the number of words across all the articles.
print ("There are {} words across all the true articles.".format(len(text)))

# Create and generate a word cloud image.
wordcloud = WordCloud(max_words=100, background_color="white").generate(text)

# Display the wordcloud.
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()
# Create a word cloud of all the fake records.
fakefilt = df_test2_filtered[(df_test2_filtered['label_bin'] == 0)]
text = " ".join(article for article in fakefilt.clean_text)

# Print the number of words across all the articles.
print ("There are {} words across all the fake articles.".format(len(text)))

# Create and generate a word cloud image.
wordcloud = WordCloud(max_words=100, background_color="white").generate(text)

# Display the wordcloud.
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output>There are 27078155 words across all the fake articles.<jupyter_text>As these wordclouds show, there is a lower word diversity across the fake articles than there is across the real articles. This is the reverse of the trend we saw with Dataset 1. It appears that many of the most popular words are consistent across both the fake news and real news articles.<jupyter_code># Distribution of the polarity of the clean text values of the fake news data. 
truefilt["polarity"] = truefilt["clean_text"].map(lambda text: TextBlob(text).sentiment.polarity)
sns.displot(truefilt["polarity"], kde=True)
fig = plt.gcf()
# Distribution of the polarity of the clean text values of the fake news data. 
fakefilt["polarity"] = fakefilt["clean_text"].map(lambda text: TextBlob(text).sentiment.polarity)
sns.displot(fakefilt["polarity"], kde=True)
fig = plt.gcf()
# Distribution of the word count of the clean text values of the news data.
df_test2_filtered["word_count"] = df_test2_filtered["clean_text"].apply(lambda x: len(str(x).split()))
sns.displot(df_test2_filtered["word_count"], kde=True)
fig = plt.gcf()<jupyter_output><empty_output><jupyter_text>Word CountsGet the Top 10 words for real and fake news datasets<jupyter_code>df_word_cnt_true = df_test2_filtered[df_test2_filtered['label']=='true']['clean_text'].str.split(expand=True).stack().value_counts().reset_index()
df_word_cnt_true.columns = ['word', 'freq']
df_word_cnt_true.head(10)
df_word_cnt_true = df_test2_filtered[df_test2_filtered['label']=='false']['clean_text'].str.split(expand=True).stack().value_counts().reset_index()
df_word_cnt_true.columns = ['word', 'freq']
df_word_cnt_true.head(10)<jupyter_output><empty_output><jupyter_text>Many of the words that appear in the top 10 for the real news dataset also appear in the top 10 for the fake news dataset.  Modeling We began by modeling Dataset 1 using Logistic, Naive Bayes, and K-Nearest Neigbors for Unigram, Bigram, and Trigram feature vector text representations.<jupyter_code>train.head(1)
start_1=time.time()<jupyter_output><empty_output><jupyter_text>Modeling with Unigram words<jupyter_code>from sklearn.feature_extraction import text ## Need to re-import because we overwrite text variable in a previous preprocessing function
corpus_train = train['clean_text']
corpus_dev = dev['clean_text']
corpus_test = test['clean_text']
y_train = train['label_bin']
y_dev = dev['label_bin']
y_test = test['label_bin']
additional_words=['reuters','washington']
stop_words = text.ENGLISH_STOP_WORDS.union(additional_words)

cv=CountVectorizer(ngram_range=(1,1),stop_words=stop_words)
X_train=cv.fit_transform(corpus_train)
X_test=cv.transform(corpus_test)

model_lr=LogisticRegression(solver="liblinear", multi_class="auto")
model_lr.fit(X_train,y_train)
y_pred=model_lr.predict(X_test)<jupyter_output><empty_output><jupyter_text>Logistic Model<jupyter_code>f1_score_1=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
print('Logistic Unigram F1-score:',f1_score_1)

weights=model_lr.coef_ #Get the weights of the logistic regression
sorted_weights=np.argsort(weights)[:,-10:].flatten() 
topic_words=cv.get_feature_names_out()[sorted_weights]  
categories=['true','fake']
topics=categories#set the topics to place in the dataframe
df_topics_lr_1=pd.DataFrame() #create dataframe
df_topics_lr_1['words']=pd.DataFrame(topic_words)
for i in range(1): 
    specific_weight=weights[i][sorted_weights] 
    df_topics_lr_1[topics[i]]=pd.DataFrame(specific_weight,columns=[topics[i]]) 
display(df_topics_lr_1) #showcase the topics and the words for the feature<jupyter_output>Logistic Unigram F1-score: 0.98418<jupyter_text>Naive Bayes Model<jupyter_code>df_alpha_stats_1=pd.DataFrame() #Create dataframe to hold alpha stats
alpha=[.001,.01,.1,1,2,5,10] #alpha-values for testing for Naive-Bayes model
for i in alpha:
    nb_stats={'Model':'Naive-Bayes','Alpha':None,'F1-Score':None} 
    nb_stats['Alpha']=i 
    nb_model=MultinomialNB(alpha=i) 
    nb_model.fit(X_train,y_train) 
    nb_pred=nb_model.predict(X_test) #Predict on dev dataset
    nb_model_f1=metrics.f1_score(y_test,nb_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    nb_stats['F1-Score']=round(nb_model_f1,4) #Make it pretty
    df_alpha_stats_1=df_alpha_stats_1.append(nb_stats,ignore_index=True)  #place the values in dataframe
print('\nShow Alpha and F1-Score for Naive-Bayes Model','\n',df_alpha_stats_1.to_string(index=False,justify='center'))

weights=nb_model.coef_ #Get the weights of the logistic regression
sorted_weights=np.argsort(weights)[:,-10:].flatten() #sort them in order to get the top 5 weights per topic
topic_words=cv.get_feature_names_out()[sorted_weights]
categories=['true','fake']
topics=categories#set the topics to place in the dataframe
df_topics_nb_1=pd.DataFrame() #create dataframe
df_topics_nb_1['words']=pd.DataFrame(topic_words) #create the column of words for the top 20 words in each category
for i in range(1): #create for loop to go through the topics
    specific_weight=weights[i][sorted_weights] #Get the specific weight for that word
    df_topics_nb_1[topics[i]]=pd.DataFrame(specific_weight,columns=[topics[i]]) #Place the word and topics in the row
display(df_topics_nb_1) #showcase the topics and the words for the feature<jupyter_output>Show Alpha and F1-Score for Naive-Bayes Model 
    Model     Alpha  F1-Score
Naive-Bayes  0.001   0.9560 
Naive-Bayes  0.010   0.9548 
Naive-Bayes  0.100   0.9506 
Naive-Bayes  1.000   0.9404 
Naive-Bayes  2.000   0.9372 
Naive-Bayes  5.000   0.9348 
Naive-Bayes 10.000   0.9348<jupyter_text>KNN Model<jupyter_code>knn_values=[1,3,5,7,9]
df_k_stats_1=pd.DataFrame()
knn_stats={'Model':'KNN','K-value':None,'F1-Score':None} #Create dictionary to hold KNN stats to append with dataframe
for i in knn_values:
    knn_stats['K-value'] = i 
    knn_model = KNeighborsClassifier(n_neighbors = i) 
    knn_model.fit(X_train,y_train) 
    knn_pred=knn_model.predict(X_test) #Predict on dev dataset 
    knn_model_f1=metrics.f1_score(y_test,knn_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    knn_stats['F1-Score'] = round(knn_model_f1,4) #Make it pretty
    df_k_stats_1 = df_k_stats_1.append(knn_stats, ignore_index=True) #place the values in dataframe
print('Show K1 and F1-Score for KNN Model','\n',df_k_stats_1.to_string(index=False,justify='center'))<jupyter_output>Show K1 and F1-Score for KNN Model 
 Model  K-value  F1-Score
 KNN    1.0      0.8025 
 KNN    3.0      0.7668 
 KNN    5.0      0.7477 
 KNN    7.0      0.7435 
 KNN    9.0      0.7356<jupyter_text>Modeling with Bigram words<jupyter_code>corpus_train = train['clean_text']
corpus_dev = dev['clean_text']
corpus_test = test['clean_text']
y_train = train['label_bin']
y_dev = dev['label_bin']
y_test = test['label_bin']
additional_words=['reuters','washington']
stop_words = text.ENGLISH_STOP_WORDS.union(additional_words)

cv_2=CountVectorizer(ngram_range=(2,2),stop_words=stop_words)
X_train=cv_2.fit_transform(corpus_train)
X_test=cv_2.transform(corpus_test)<jupyter_output><empty_output><jupyter_text>Logistic Model<jupyter_code>model_lr_2=LogisticRegression(solver="liblinear", multi_class="auto")
model_lr_2.fit(X_train,y_train)
y_pred=model_lr_2.predict(X_test)
f1_score_2=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
print('Logistic Bigram F1-Score',f1_score_2)<jupyter_output>Logistic Bigram F1-Score 0.97122<jupyter_text>Naive Bayes Model<jupyter_code>df_alpha_stats_2=pd.DataFrame() #Create dataframe to hold alpha stats
alpha=[.001,.01,.1,1,2,5,10] #alpha-values for testing for Naive-Bayes model
for i in alpha:
    nb_stats={'Model':'Naive-Bayes','Alpha':None,'F1-Score':None} 
    nb_stats['Alpha']=i 
    nb_model=MultinomialNB(alpha=i) 
    nb_model.fit(X_train,y_train) 
    nb_pred=nb_model.predict(X_test) #Predict on dev dataset
    nb_model_f1=metrics.f1_score(y_test,nb_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    nb_stats['F1-Score']=round(nb_model_f1,4) #Make it pretty
    df_alpha_stats_2=df_alpha_stats_2.append(nb_stats,ignore_index=True)  #place the values in dataframe
print('\nShow Alpha and F1-Score for Naive-Bayes Model','\n',df_alpha_stats_2.to_string(index=False,justify='center'))

weights=nb_model.coef_ #Get the weights of the logistic regression
sorted_weights=np.argsort(weights)[:,-10:].flatten() #sort them in order to get the top 5 weights per topic
topic_words=cv_2.get_feature_names_out()[sorted_weights] #Grab the words from the top 5 weights 
categories=['true','fake']
topics=categories#set the topics to place in the dataframe
df_topics_nb_2=pd.DataFrame() #create dataframe
df_topics_nb_2['words']=pd.DataFrame(topic_words) #create the column of words for the top 20 words in each category
for i in range(1): #create for loop to go through the topics
    print(i)
    specific_weight=weights[i][sorted_weights] #Get the specific weight for that word
    df_topics_nb_2[topics[i]]=pd.DataFrame(specific_weight,columns=[topics[i]]) #Place the word and topics in the row
display(df_topics_nb_2) #showcase the topics and the words for the feature<jupyter_output>Show Alpha and F1-Score for Naive-Bayes Model 
    Model     Alpha  F1-Score
Naive-Bayes  0.001   0.9649 
Naive-Bayes  0.010   0.9664 
Naive-Bayes  0.100   0.9688 
Naive-Bayes  1.000   0.9720 
Naive-Bayes  2.000   0.9703 
Naive-Bayes  5.000   0.9685 
Naive-Bayes 10.000   0.9661 
0<jupyter_text>KNN Model<jupyter_code>knn_values=[1,3,5,7,9]
df_k_stats_2=pd.DataFrame()
knn_stats={'Model':'KNN','K-value':None,'F1-Score':None} #Create dictionary to hold KNN stats to append with dataframe
for i in knn_values:
    knn_stats['K-value'] = i 
    knn_model = KNeighborsClassifier(n_neighbors = i) 
    knn_model.fit(X_train,y_train) 
    knn_pred=knn_model.predict(X_test) #Predict on dev dataset 
    knn_model_f1=metrics.f1_score(y_test,knn_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    knn_stats['F1-Score'] = round(knn_model_f1,5) #Make it pretty
    df_k_stats_2 = df_k_stats_2.append(knn_stats, ignore_index=True) #place the values in dataframe
print('Show K1 and F1-Score for KNN Model','\n',df_k_stats_2.to_string(index=False,justify='center'))<jupyter_output>Show K1 and F1-Score for KNN Model 
 Model  K-value  F1-Score
 KNN    1.0     0.50637 
 KNN    3.0     0.42563 
 KNN    5.0     0.40525 
 KNN    7.0     0.39125 
 KNN    9.0     0.38755<jupyter_text>Modeling with Trigram words<jupyter_code>corpus_train = train['clean_text']
corpus_dev = dev['clean_text']
corpus_test = test['clean_text']
y_train = train['label_bin']
y_dev = dev['label_bin']
y_test = test['label_bin']
additional_words=['reuters','washington']
stop_words = text.ENGLISH_STOP_WORDS.union(additional_words)
cv_3=CountVectorizer(ngram_range=(3,3),stop_words=stop_words)
X_train=cv_3.fit_transform(corpus_train)
X_test=cv_3.transform(corpus_test)<jupyter_output><empty_output><jupyter_text>Logistic Model<jupyter_code>model_lr_3=LogisticRegression(solver="liblinear", multi_class="auto")
model_lr_3.fit(X_train,y_train)
y_pred=model_lr_3.predict(X_test)
f1_score_3=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
print('Logistic Trigram Word F1-Score',f1_score_3)<jupyter_output>Logistic Trigram Word F1-Score 0.91292<jupyter_text>Naive-Bayes Model<jupyter_code>df_alpha_stats_3=pd.DataFrame() #Create dataframe to hold alpha stats
alpha=[.001,.01,.1,1,2,5,10] #alpha-values for testing for Naive-Bayes model
for i in alpha:
    nb_stats={'Model':'Naive-Bayes','Alpha':None,'F1-Score':None} 
    nb_stats['Alpha']=i 
    nb_model=MultinomialNB(alpha=i) 
    nb_model.fit(X_train,y_train) 
    nb_pred=nb_model.predict(X_test) #Predict on dev dataset
    nb_model_f1=metrics.f1_score(y_test,nb_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    nb_stats['F1-Score']=round(nb_model_f1,4) #Make it pretty
    df_alpha_stats_3=df_alpha_stats_3.append(nb_stats,ignore_index=True)  #place the values in dataframe
print('\nShow Alpha and F1-Score for Naive-Bayes Model','\n',df_alpha_stats_3.to_string(index=False,justify='center'))

weights=nb_model.coef_ 
sorted_weights=np.argsort(weights)[:,-10:].flatten() 
topic_words=cv_3.get_feature_names_out()[sorted_weights] 
categories=['true','fake']
topics=categories#set the topics to place in the dataframe
df_topics_nb_3=pd.DataFrame() #create dataframe
df_topics_nb_3['words']=pd.DataFrame(topic_words) #create the column of words for the top 20 words in each category
for i in range(1): #create for loop to go through the topics
    print(i)
    specific_weight=weights[i][sorted_weights] #Get the specific weight for that word
    df_topics_nb_3[topics[i]]=pd.DataFrame(specific_weight,columns=[topics[i]]) #Place the word and topics in the row
display(df_topics_nb_3)<jupyter_output>Show Alpha and F1-Score for Naive-Bayes Model 
    Model     Alpha  F1-Score
Naive-Bayes  0.001   0.9554 
Naive-Bayes  0.010   0.9571 
Naive-Bayes  0.100   0.9586 
Naive-Bayes  1.000   0.9586 
Naive-Bayes  2.000   0.9590 
Naive-Bayes  5.000   0.9558 
Naive-Bayes 10.000   0.9528 
0<jupyter_text>KNN Model<jupyter_code>knn_values=[1,3,5,7,9]
df_k_stats_3=pd.DataFrame()
knn_stats={'Model':'KNN','K-value':None,'F1-Score':None} #Create dictionary to hold KNN stats to append with dataframe
for i in knn_values:
    knn_stats['K-value'] = i 
    knn_model = KNeighborsClassifier(n_neighbors = i) 
    knn_model.fit(X_train,y_train) 
    knn_pred=knn_model.predict(X_test) #Predict on dev dataset 
    knn_model_f1=metrics.f1_score(y_test,knn_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    knn_stats['F1-Score'] = round(knn_model_f1,4) #Make it pretty
    df_k_stats_3 = df_k_stats_3.append(knn_stats, ignore_index=True) #place the values in dataframe
print('Show K1 and F1-Score for KNN Model','\n',df_k_stats_3.to_string(index=False,justify='center'))<jupyter_output>Show K1 and F1-Score for KNN Model 
 Model  K-value  F1-Score
 KNN    1.0      0.4091 
 KNN    3.0      0.3593 
 KNN    5.0      0.3524 
 KNN    7.0      0.3487 
 KNN    9.0      0.3481<jupyter_text>Model Analysis based on Max Features - Dataset 1<jupyter_code>corpus_train = train['clean_text']
corpus_dev = dev['clean_text']
corpus_test = test['clean_text']
y_train = train['label_bin']
y_dev = dev['label_bin']
y_test = test['label_bin']
additional_words=['reuters','washington']
stop_words = text.ENGLISH_STOP_WORDS.union(additional_words)

max_features=[10,100,1000,10000,100000]
df_lr_performance=pd.DataFrame()
lr_perf={'Model':'Logistic Regression','Features Count':None,'F1-Score':None,'Performance Time':None}
for i in max_features:
    lr_perf['Features Count']=i
    start = time.time()
    cv=CountVectorizer(ngram_range=(1,1),stop_words=stop_words,max_features=i)
    X_train=cv.fit_transform(corpus_train)
    X_test=cv.transform(corpus_test)
    model_lr=LogisticRegression(solver="liblinear", multi_class="auto")
    model_lr.fit(X_train,y_train)
    y_pred=model_lr.predict(X_test)
    duration=time.time()-start
    f1_score=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
    lr_perf['F1-Score']=f1_score
    lr_perf['Performance Time']=duration
    df_lr_performance = df_lr_performance.append(lr_perf, ignore_index=True)
    print(i,duration,f1_score)

#Adding the final performance after 100K features have been added
lr_perf={'Model':'Logistic Regression','Features Count':None,'F1-Score':None,'Performance Time':None}
start = time.time()
cv=CountVectorizer(ngram_range=(1,1),stop_words=stop_words)
X_train=cv.fit_transform(corpus_train)
X_test=cv.transform(corpus_test)
model_lr=LogisticRegression(solver="liblinear", multi_class="auto")
model_lr.fit(X_train,y_train)
y_pred=model_lr.predict(X_test)
duration=time.time()-start
f1_score=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
lr_perf['F1-Score']=f1_score
lr_perf['Performance Time']=duration
lr_perf['Features Count']=len(cv.get_feature_names_out())
df_lr_performance = df_lr_performance.append(lr_perf, ignore_index=True)
df_lr_performance['Features Count']=df_lr_performance['Features Count'].astype(int)

#Graphing and Display Performance with Feature Size

df_lr_performance.plot.line(x='Features Count',y='F1-Score',ylabel='Model F1-Score',title='Performance based on Vocabulary Size',logx=True,color='purple') #Create plot with purple because purple is cool.  
df_lr_performance.plot.line(x='Performance Time',y='F1-Score',ylabel='Model F1-Score',title='Performance based on Duration',logx=False,color='purple') #Create plot with purple because purple is cool.  
display(df_lr_performance)<jupyter_output><empty_output><jupyter_text>F1 Score Analysis with all Models - Dataset 1<jupyter_code>df_all_models=pd.DataFrame()
logistic_models={'Model':None,'Special Characteristic':None,'F1-Score':None}
log_models={'Logistic Model Unigram':f1_score_1,'Logistic Model Bigram':f1_score_2,'Logistic Model Trigram':f1_score_3}
knn_models=[df_k_stats_1,df_k_stats_2,df_k_stats_3]
for i in log_models:
    logistic_models['Model']=i
    logistic_models['F1-Score']=log_models[i]
    df_all_models=df_all_models.append(logistic_models, ignore_index=True)
nb_models=[df_alpha_stats_1,df_alpha_stats_2,df_alpha_stats_3]
count=0
for i in nb_models:
    max_value_1=i.loc[i['F1-Score'].idxmax()]
    if count==0:
        max_value_1=max_value_1.replace(to_replace='Naive-Bayes',value='Naive-Bayes Unigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue
    if count==1:
        max_value_1=max_value_1.replace(to_replace='Naive-Bayes',value='Naive-Bayes Bigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)

        continue
    if count==2:
        max_value_1=max_value_1.replace(to_replace='Naive-Bayes',value='Naive-Bayes Trigram')
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue
count=0
for i in knn_models:
    max_value_1=i.loc[i['F1-Score'].idxmax()]
    if count==0:
        max_value_1=max_value_1.replace(to_replace='KNN',value='KNN Unigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue
    if count==1:
        max_value_1=max_value_1.replace(to_replace='KNN',value='KNN Bigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)

        continue
    if count==2:
        max_value_1=max_value_1.replace(to_replace='KNN',value='KNN Trigram')
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue

df_all_models['Special Characteristic']=df_all_models['Special Characteristic'].combine_first(df_all_models['Alpha'])
df_all_models['Special Characteristic']=df_all_models['Special Characteristic'].combine_first(df_all_models['K-value'])
df_all_models=df_all_models.drop(columns=['Alpha', 'K-value'])
display(df_all_models)

# Plot F1 score of all models.
df_all_models.plot.bar(x='Model',y='F1-Score',color=['purple','green','orange'],title='F1-Score All Models',legend=False)

# Print time it takes to run all models.
end_1=time.time()
duration=((start_1-end_1)*-1)/60
print(f'The total time it takes for all of modeling to finish is {duration} minutes')<jupyter_output>The total time it takes for all of modeling to finish is 9.803922533988953 minutes<jupyter_text>Evaluate Models based on Dataset 2  Evaluation with Unigram words<jupyter_code>## Need to re-import because we overwrite text variable in a previous preprocessing function
from sklearn.feature_extraction import text

corpus_train = train['clean_text']
corpus_test = df_test2_filtered['clean_text']
y_train = train['label_bin']
y_test = df_test2_filtered['label_bin']

additional_words=['reuters','washington']
stop_words = text.ENGLISH_STOP_WORDS.union(additional_words)

cv=CountVectorizer(ngram_range=(1,1),stop_words=stop_words)
X_train=cv.fit_transform(corpus_train)
X_test=cv.transform(corpus_test)

model_lr=LogisticRegression(solver="liblinear", multi_class="auto")
model_lr.fit(X_train,y_train)
y_pred=model_lr.predict(X_test)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>f1_score_1=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
print('Logistic Unigram F1-score:',f1_score_1)<jupyter_output>Logistic Unigram F1-score: 0.52416<jupyter_text>Naive Bayes<jupyter_code>df_alpha_stats_1=pd.DataFrame() #Create dataframe to hold alpha stats
alpha=[.001,.01,.1,1,2,5,10] #alpha-values for testing for Naive-Bayes model
for i in alpha:
    nb_stats={'Model':'Naive-Bayes','Alpha':None,'F1-Score':None} 
    nb_stats['Alpha']=i 
    nb_model=MultinomialNB(alpha=i) 
    nb_model.fit(X_train,y_train) 
    nb_pred=nb_model.predict(X_test) #Predict on dev dataset
    nb_model_f1=metrics.f1_score(y_test,nb_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    nb_stats['F1-Score']=round(nb_model_f1,4) #Make it pretty
    df_alpha_stats_1=df_alpha_stats_1.append(nb_stats,ignore_index=True)  #place the values in dataframe
print('\nShow Alpha and F1-Score for Naive-Bayes Model','\n',df_alpha_stats_1.to_string(index=False,justify='center'))<jupyter_output>Show Alpha and F1-Score for Naive-Bayes Model 
    Model     Alpha  F1-Score
Naive-Bayes  0.001   0.5350 
Naive-Bayes  0.010   0.5354 
Naive-Bayes  0.100   0.5357 
Naive-Bayes  1.000   0.5387 
Naive-Bayes  2.000   0.5377 
Naive-Bayes  5.000   0.5331 
Naive-Bayes 10.000   0.5285<jupyter_text>KNN<jupyter_code>knn_values=[1,3,5,7,9]
df_k_stats_1=pd.DataFrame()
knn_stats={'Model':'KNN','K-value':None,'F1-Score':None} #Create dictionary to hold KNN stats to append with dataframe
for i in knn_values:
    knn_stats['K-value'] = i 
    knn_model = KNeighborsClassifier(n_neighbors = i) 
    knn_model.fit(X_train,y_train) 
    knn_pred=knn_model.predict(X_test) #Predict on dev dataset 
    knn_model_f1=metrics.f1_score(y_test,knn_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    knn_stats['F1-Score'] = round(knn_model_f1,4) #Make it pretty
    df_k_stats_1 = df_k_stats_1.append(knn_stats, ignore_index=True) #place the values in dataframe
print('Show K1 and F1-Score for KNN Model','\n',df_k_stats_1.to_string(index=False,justify='center'))<jupyter_output>Show K1 and F1-Score for KNN Model 
 Model  K-value  F1-Score
 KNN    1.0      0.6209 
 KNN    3.0      0.6230 
 KNN    5.0      0.6361 
 KNN    7.0      0.6361 
 KNN    9.0      0.6398<jupyter_text>Evaluation with Bigram words<jupyter_code>corpus_train = train['clean_text']
corpus_test = df_test2_filtered['clean_text']
y_train = train['label_bin']
y_test = df_test2_filtered['label_bin']
additional_words=['reuters','washington']
stop_words = text.ENGLISH_STOP_WORDS.union(additional_words)

cv_2=CountVectorizer(ngram_range=(2,2),stop_words=stop_words)
X_train=cv_2.fit_transform(corpus_train)
X_test=cv_2.transform(corpus_test)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>model_lr_2=LogisticRegression(solver="liblinear", multi_class="auto")
model_lr_2.fit(X_train,y_train)
y_pred=model_lr_2.predict(X_test)
f1_score_2=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
print('Logistic Bigram F1-Score',f1_score_2)<jupyter_output>Logistic Bigram F1-Score 0.56845<jupyter_text>Naive Bayes<jupyter_code>df_alpha_stats_2=pd.DataFrame() #Create dataframe to hold alpha stats
alpha=[.001,.01,.1,1,2,5,10] #alpha-values for testing for Naive-Bayes model
for i in alpha:
    nb_stats={'Model':'Naive-Bayes','Alpha':None,'F1-Score':None} 
    nb_stats['Alpha']=i 
    nb_model=MultinomialNB(alpha=i) 
    nb_model.fit(X_train,y_train) 
    nb_pred=nb_model.predict(X_test) #Predict on dev dataset
    nb_model_f1=metrics.f1_score(y_test,nb_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    nb_stats['F1-Score']=round(nb_model_f1,4) #Make it pretty
    df_alpha_stats_2=df_alpha_stats_2.append(nb_stats,ignore_index=True)  #place the values in dataframe
print('\nShow Alpha and F1-Score for Naive-Bayes Model','\n',df_alpha_stats_2.to_string(index=False,justify='center'))<jupyter_output>Show Alpha and F1-Score for Naive-Bayes Model 
    Model     Alpha  F1-Score
Naive-Bayes  0.001   0.5728 
Naive-Bayes  0.010   0.5753 
Naive-Bayes  0.100   0.5806 
Naive-Bayes  1.000   0.5820 
Naive-Bayes  2.000   0.5806 
Naive-Bayes  5.000   0.5822 
Naive-Bayes 10.000   0.5837<jupyter_text>KNN<jupyter_code>knn_values=[1,3,5,7,9]
df_k_stats_2=pd.DataFrame()
knn_stats={'Model':'KNN','K-value':None,'F1-Score':None} #Create dictionary to hold KNN stats to append with dataframe
for i in knn_values:
    knn_stats['K-value'] = i 
    knn_model = KNeighborsClassifier(n_neighbors = i) 
    knn_model.fit(X_train,y_train) 
    knn_pred=knn_model.predict(X_test) #Predict on dev dataset 
    knn_model_f1=metrics.f1_score(y_test,knn_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    knn_stats['F1-Score'] = round(knn_model_f1,5) #Make it pretty
    df_k_stats_2 = df_k_stats_2.append(knn_stats, ignore_index=True) #place the values in dataframe
print('Show K1 and F1-Score for KNN Model','\n',df_k_stats_2.to_string(index=False,justify='center'))<jupyter_output>Show K1 and F1-Score for KNN Model 
 Model  K-value  F1-Score
 KNN    1.0     0.38926 
 KNN    3.0     0.37464 
 KNN    5.0     0.36777 
 KNN    7.0     0.36312 
 KNN    9.0     0.36013<jupyter_text>Evaluation with Trigram words<jupyter_code>corpus_train = train['clean_text']
corpus_test = df_test2_filtered['clean_text']
y_train = train['label_bin']
y_test = df_test2_filtered['label_bin']

additional_words=['reuters','washington']
stop_words = text.ENGLISH_STOP_WORDS.union(additional_words)
cv_3=CountVectorizer(ngram_range=(3,3),stop_words=stop_words)
X_train=cv_3.fit_transform(corpus_train)
X_test=cv_3.transform(corpus_test)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>model_lr_3=LogisticRegression(solver="liblinear", multi_class="auto")
model_lr_3.fit(X_train,y_train)
y_pred=model_lr_3.predict(X_test)
f1_score_3=round(metrics.f1_score(y_test,y_pred,average="weighted"),5)
print('Logistic Trigram Word F1-Score',f1_score_3)<jupyter_output>Logistic Trigram Word F1-Score 0.53931<jupyter_text>Naive Bayes<jupyter_code>df_alpha_stats_3=pd.DataFrame() #Create dataframe to hold alpha stats
alpha=[.001,.01,.1,1,2,5,10] #alpha-values for testing for Naive-Bayes model
for i in alpha:
    nb_stats={'Model':'Naive-Bayes','Alpha':None,'F1-Score':None} 
    nb_stats['Alpha']=i 
    nb_model=MultinomialNB(alpha=i) 
    nb_model.fit(X_train,y_train) 
    nb_pred=nb_model.predict(X_test) #Predict on dev dataset
    nb_model_f1=metrics.f1_score(y_test,nb_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    nb_stats['F1-Score']=round(nb_model_f1,4) #Make it pretty
    df_alpha_stats_3=df_alpha_stats_3.append(nb_stats,ignore_index=True)  #place the values in dataframe
print('\nShow Alpha and F1-Score for Naive-Bayes Model','\n',df_alpha_stats_3.to_string(index=False,justify='center'))<jupyter_output>Show Alpha and F1-Score for Naive-Bayes Model 
    Model     Alpha  F1-Score
Naive-Bayes  0.001   0.6027 
Naive-Bayes  0.010   0.6055 
Naive-Bayes  0.100   0.6072 
Naive-Bayes  1.000   0.6100 
Naive-Bayes  2.000   0.6136 
Naive-Bayes  5.000   0.6189 
Naive-Bayes 10.000   0.6229<jupyter_text>KNN<jupyter_code>knn_values=[1,3,5,7,9]
df_k_stats_3=pd.DataFrame()
knn_stats={'Model':'KNN','K-value':None,'F1-Score':None} #Create dictionary to hold KNN stats to append with dataframe
for i in knn_values:
    knn_stats['K-value'] = i 
    knn_model = KNeighborsClassifier(n_neighbors = i) 
    knn_model.fit(X_train,y_train) 
    knn_pred=knn_model.predict(X_test) #Predict on dev dataset 
    knn_model_f1=metrics.f1_score(y_test,knn_pred,average="weighted") #Get f1-score based on the dev labels and predicted values
    knn_stats['F1-Score'] = round(knn_model_f1,4) #Make it pretty
    df_k_stats_3 = df_k_stats_3.append(knn_stats, ignore_index=True) #place the values in dataframe
print('Show K1 and F1-Score for KNN Model','\n',df_k_stats_3.to_string(index=False,justify='center'))<jupyter_output>Show K1 and F1-Score for KNN Model 
 Model  K-value  F1-Score
 KNN    1.0      0.3126 
 KNN    3.0      0.3118 
 KNN    5.0      0.3118 
 KNN    7.0      0.3116 
 KNN    9.0      0.3115<jupyter_text>F1 Score Analysis with all Models - Dataset 2<jupyter_code>df_all_models=pd.DataFrame()
logistic_models={'Model':None,'Special Characteristic':None,'F1-Score':None}
log_models={'Logistic Model Unigram':f1_score_1,'Logistic Model Bigram':f1_score_2,'Logistic Model Trigram':f1_score_3}
knn_models=[df_k_stats_1,df_k_stats_2,df_k_stats_3]
for i in log_models:
    logistic_models['Model']=i
    logistic_models['F1-Score']=log_models[i]
    df_all_models=df_all_models.append(logistic_models, ignore_index=True)
nb_models=[df_alpha_stats_1,df_alpha_stats_2,df_alpha_stats_3]
count=0
for i in nb_models:
    max_value_1=i.loc[i['F1-Score'].idxmax()]
    if count==0:
        max_value_1=max_value_1.replace(to_replace='Naive-Bayes',value='Naive-Bayes Unigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue
    if count==1:
        max_value_1=max_value_1.replace(to_replace='Naive-Bayes',value='Naive-Bayes Bigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)

        continue
    if count==2:
        max_value_1=max_value_1.replace(to_replace='Naive-Bayes',value='Naive-Bayes Trigram')
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue
count=0
for i in knn_models:
    max_value_1=i.loc[i['F1-Score'].idxmax()]
    if count==0:
        max_value_1=max_value_1.replace(to_replace='KNN',value='KNN Unigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue
    if count==1:
        max_value_1=max_value_1.replace(to_replace='KNN',value='KNN Bigram')
        count+=1
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)

        continue
    if count==2:
        max_value_1=max_value_1.replace(to_replace='KNN',value='KNN Trigram')
        df_all_models=df_all_models.append(max_value_1,ignore_index=True)
        continue

df_all_models['Special Characteristic']=df_all_models['Special Characteristic'].combine_first(df_all_models['Alpha'])
df_all_models['Special Characteristic']=df_all_models['Special Characteristic'].combine_first(df_all_models['K-value'])
df_all_models=df_all_models.drop(columns=['Alpha', 'K-value'])
display(df_all_models)
# Plot F1 score of all models.
df_all_models.plot.bar(x='Model',y='F1-Score',color=['purple','green','orange'],title='F1-Score All Models',legend=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12345703473281687, 'fraction_numerical': 0.031423986051395854, 'mean_word_length': 5.975124378109452, 'pattern_counts': {'":': 0, '<': 146, '<?xml version=': 0, '>': 147, 'https://': 2, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '41', 'dir': 'jupyter-structured-clean-dedup', 'id': '333397', 'license': 'MIT', 'n_tokens_mistral': 18852, 'n_tokens_neox': 17909, 'n_words': 3981, 'path': 'final_submission.ipynb', 'repo_name': 'akikoiwamizu/w207-project', 'size': '50493'}