<jupyter_start><jupyter_text>Logistic Regression Model Video ClassifierThis notebook uses Logistic Regression to predict a Youtube category based off of the title. We will run this on three inputs. Filtered titles, filtered and normalized titles, andfiltered titles with no stop words<jupyter_code>import pandas as pd
from nltk.corpus import stopwords

from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn import metrics
from sklearn.metrics import accuracy_score, confusion_matrix
import matplotlib.pyplot as plt
from yellowbrick.classifier import ConfusionMatrix, ClassificationReport


import string

from sklearn.linear_model import LogisticRegression

import warnings
warnings.filterwarnings("ignore", category=FutureWarning)<jupyter_output><empty_output><jupyter_text>Load data and get DataHere, we want to load the csv file and separate each column.<jupyter_code>df_dataset = pd.read_csv('./output/US_count_vectorizer_dataset.csv')
df_dataset.head()<jupyter_output><empty_output><jupyter_text>1. df_x are regular, filtered title2. df_x2 are filtered title that does not contain stop words3. df_y are video categories4. target_names are all the possible categories<jupyter_code>df_x = df_dataset['filter_title']
df_x2 = df_dataset['filter_title_no_stops']
df_y = df_dataset['category_id']
target_names = list(df_dataset['category_id'].unique())<jupyter_output><empty_output><jupyter_text>Split the dataWe want to split the data here where 80 percent are reserved for training and 20 percent is reserved for testing<jupyter_code>x_train, x_test, y_train, y_test = train_test_split(df_x, df_y, test_size=0.2, random_state=117)

x_train2, x_test2 = train_test_split(df_x2, test_size=0.2, random_state=117)

# replace NaN with spaces so count vectorizer can work
x_train2 = x_train2.fillna(' ')
x_test2 = x_test2.fillna(' ')<jupyter_output><empty_output><jupyter_text>Tokenize wordsHere, we want to essentially want to convert our words in the title here into 1 and 0sso the Logistic Regression model can recognize it<jupyter_code>count_vectorizer = CountVectorizer()
train_count_vector = count_vectorizer.fit_transform(x_train)
test_count_vector = count_vectorizer.transform(x_test)
print(train_count_vector.shape)
print(train_count_vector.toarray())


train_count_vector2 = count_vectorizer.fit_transform(x_train2)
test_count_vector2 = count_vectorizer.transform(x_test2)<jupyter_output>(32759, 10574)
[[0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 ...
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]
 [0 0 0 ... 0 0 0]]<jupyter_text>Normalize filtered title<jupyter_code>tfidf_vectorizer = TfidfTransformer()
x_trained_tfidf_vector = tfidf_vectorizer.fit_transform(train_count_vector)
x_test_tfidf_vector = tfidf_vectorizer.transform(test_count_vector)
print(x_trained_tfidf_vector.shape)
print(x_trained_tfidf_vector.toarray())<jupyter_output>(32759, 10574)
[[0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 ...
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]]<jupyter_text>Train Logistic Regression Model and use to predict1. Here we want to use our previously split training data to train the logistic model.2. Afterwards, we will feed it our testing test and print out an accuracy report.<jupyter_code>LogReg = LogisticRegression(solver='lbfgs', max_iter = 4000)
normalizedLogRegFit = LogReg.fit(x_trained_tfidf_vector, y_train)
normalized_predict = normalizedLogRegFit.predict(x_test_tfidf_vector)
print("Normalized Accuracy: ", accuracy_score(y_test, normalized_predict))
print(metrics.classification_report(y_test, normalized_predict, target_names=target_names))

normalized_accuracy = accuracy_score(y_test, normalized_predict)
LogReg2 = LogisticRegression(solver='lbfgs', max_iter = 4000)
RegularLogRegFit = LogReg2.fit(train_count_vector, y_train)
regular_predict = RegularLogRegFit.predict(test_count_vector)
print("Regular Title Accuracy: ", accuracy_score(y_test, regular_predict))
print(metrics.classification_report(y_test, regular_predict, target_names=target_names))

regular_accuracy = accuracy_score(y_test, regular_predict)
LogReg3 = LogisticRegression(solver='lbfgs', max_iter = 4000)

stopWordsLogRegFit = LogReg3.fit(train_count_vector2, y_train)
stopWords_predict = stopWordsLogRegFit.predict(test_count_vector2)
print("No Stopwords Title Accuracy: ", accuracy_score(y_test, stopWords_predict))
print(metrics.classification_report(y_test, stopWords_predict, target_names=target_names))

stopwords_accuracy = accuracy_score(y_test, stopWords_predict)
fig = plt.figure()
fig.set_size_inches(5, 5)
ax = fig.add_axes([0,0,1,1])
x = ["Regular Filtered", "Filtered/No StopWords", "Normalized Filtered"]
y = [regular_accuracy,stopwords_accuracy, normalized_accuracy]
fig.suptitle('LogReg Input vs Accuracy', fontsize=10)
plt.xlabel('Input Type', fontsize=15)
plt.ylabel('Accuracy Percentage', fontsize=14)
ax.bar(x,y)
plt.show()
myLogisticRegression = LogisticRegression(solver='lbfgs', max_iter = 4000)
conf_matrix = ConfusionMatrix(myLogisticRegression, cmapstring='YlOrRd', percent=True, cmap='Greens')
conf_matrix.fit(train_count_vector, y_train)
conf_matrix.score(test_count_vector, y_test)
conf_matrix.show()
myLogisticRegression2 = LogisticRegression(solver='lbfgs', max_iter = 4000)
conf_matrix = ConfusionMatrix(myLogisticRegression2, cmapstring='YlOrRd', percent=True, cmap='Greens')
conf_matrix.fit(x_trained_tfidf_vector, y_train)
conf_matrix.score(x_test_tfidf_vector, y_test)
conf_matrix.show()
myLogisticRegression3 = LogisticRegression(solver='lbfgs', max_iter = 4000)
conf_matrix = ConfusionMatrix(myLogisticRegression3, cmapstring='YlOrRd', percent=True, cmap='Greens')
conf_matrix.fit(train_count_vector2, y_train)
conf_matrix.score(test_count_vector2, y_test)
conf_matrix.show()
myLogisticRegression4 = LogisticRegression(solver='lbfgs', max_iter = 4000)
visualizer = ClassificationReport(myLogisticRegression4, support=True, cmap='Greens')
visualizer.fit(x_trained_tfidf_vector, y_train)
visualizer.score(x_test_tfidf_vector, y_test)
visualizer.show()
myLogisticRegression5 = LogisticRegression(solver='lbfgs', max_iter = 4000)
visualizer = ClassificationReport(myLogisticRegression5, support=True, cmap='Greens')
visualizer.fit(train_count_vector, y_train)
visualizer.score(test_count_vector, y_test)
visualizer.finalize(set_title="this")
myLogisticRegression6 = LogisticRegression(solver='lbfgs', max_iter = 4000)
visualizer = ClassificationReport(myLogisticRegression6, support=True, cmap='Greens')
visualizer.fit(train_count_vector2, y_train)
visualizer.score(test_count_vector2, y_test)
visualizer.finalize(set_title="this")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10093374833259226, 'fraction_numerical': 0.028901734104046242, 'mean_word_length': 7.925925925925926, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '533208', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 2469, 'n_tokens_neox': 2287, 'n_words': 503, 'path': 'Logistic Regression Model.ipynb', 'repo_name': 'Vic-the-Legend/yt-trends', 'size': '6747'}