<jupyter_start><jupyter_text>Six buckets<jupyter_code>train_data_target_6k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/train_6_buckets.csv')
test_data_target_6k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/test_6_buckets.csv')
val_data_target_6k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/valid_6_buckets.csv')
testData['data_IMDBscore']=test_data_target_6k['data_IMDBscore']
trainData['data_IMDBscore']=train_data_target_6k['data_IMDBscore']
validData ['data_IMDBscore']=val_data_target_6k['data_IMDBscore']
train_X = trainData.drop(columns=['data_IMDBscore'])
train_Y = trainData['data_IMDBscore']
test_X = testData.drop(columns=['data_IMDBscore'])
test_Y = testData['data_IMDBscore']
val_X=validData.drop(columns=['data_IMDBscore'])
val_Y=validData['data_IMDBscore']
#Import Gaussian Naive Bayes model
from sklearn.naive_bayes import GaussianNB

#Create a Gaussian Classifier
model = GaussianNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)

filename = '/content/gdrive/MyDrive/SavedModels/GNB_6_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))
#Import Multinomial Naive Bayes model
from sklearn.naive_bayes import MultinomialNB

#Create a Multinomial Classifier
model = MultinomialNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)<jupyter_output><empty_output><jupyter_text>Saving Model<jupyter_code>filename = '/content/gdrive/MyDrive/SavedModels/MNB_6_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))
#Import Bernoulli Naive Bayes model
from sklearn.naive_bayes import BernoulliNB

#Create a Bernoulli Classifier
model = BernoulliNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)

filename = '/content/gdrive/MyDrive/SavedModels/BNB_6_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))
import pandas as pd
from google.colab import drive

drive.mount('/content/gdrive', force_remount=True)
import pandas as pd
import numpy as np
testData = pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/pre_standardization_test.csv')
trainData = pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/pre_standardization_train.csv')
validData = pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/pre_standardization_val.csv')
#Min Max Scaling
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
scaler.fit(testData)
scaler.fit(trainData)
scaler.fit(validData )<jupyter_output><empty_output><jupyter_text>Eleven Buckets<jupyter_code>train_data_target_11k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/train_11_buckets.csv')
test_data_target_11k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/test_11_buckets.csv')
val_data_target_11k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/valid_11_buckets.csv')
testData['data_IMDBscore']=test_data_target_11k['data_IMDBscore']
trainData['data_IMDBscore']=train_data_target_11k['data_IMDBscore']
validData ['data_IMDBscore']=val_data_target_11k['data_IMDBscore']
train_X = trainData.drop(columns=['data_IMDBscore'])
train_Y = trainData['data_IMDBscore']
test_X = testData.drop(columns=['data_IMDBscore'])
test_Y = testData['data_IMDBscore']
val_X=validData.drop(columns=['data_IMDBscore'])
val_Y=validData['data_IMDBscore']
#Import Gaussian Naive Bayes model
from sklearn.naive_bayes import GaussianNB

#Create a Gaussian Classifier
model = GaussianNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)

filename = '/content/gdrive/MyDrive/SavedModels/GNB_11_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))
#Import Multinomial Naive Bayes model
from sklearn.naive_bayes import MultinomialNB

#Create a Multinomial Classifier
model = MultinomialNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)

filename = '/content/gdrive/MyDrive/SavedModels/MNB_11_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))
#Import Bernoulli Naive Bayes model
from sklearn.naive_bayes import BernoulliNB

#Create a Bernoulli Classifier
model = BernoulliNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)

filename = '/content/gdrive/MyDrive/SavedModels/BNB_11_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))<jupyter_output>test_f1_score 0.40427661271381266
test_precision_score 0.40257973765831695
test_recall_score 0.4121787025703794
test_accuracy_score 0.4121787025703794<jupyter_text>TwentyOne Buckets<jupyter_code>train_data_target_21k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/train_21_buckets.csv')
test_data_target_21k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/test_21_buckets.csv')
val_data_target_21k=pd.read_csv('/content/gdrive/MyDrive/ML_Project/Dataset/valid_21_buckets.csv')
testData['data_IMDBscore']=test_data_target_21k['data_IMDBscore']
trainData['data_IMDBscore']=train_data_target_21k['data_IMDBscore']
validData ['data_IMDBscore']=val_data_target_21k['data_IMDBscore']
train_X = trainData.drop(columns=['data_IMDBscore'])
train_Y = trainData['data_IMDBscore']*2
test_X = testData.drop(columns=['data_IMDBscore'])
test_Y = testData['data_IMDBscore']*2
val_X=validData.drop(columns=['data_IMDBscore'])
val_Y=validData['data_IMDBscore']*2
#Import Gaussian Naive Bayes model
from sklearn.naive_bayes import GaussianNB

#Create a Gaussian Classifier
model = GaussianNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)




filename = '/content/gdrive/MyDrive/SavedModels/GNB_21_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))
#Import Multinomial Naive Bayes model
from sklearn.naive_bayes import MultinomialNB

#Create a Multinomial Classifier
model = MultinomialNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test dataset
y_pred = model.predict(test_X)
#Import scikit-learn metrics module for accuracy calculation

filename = '/content/gdrive/MyDrive/SavedModels/MNB_21_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))
#Import Bernoulli Naive Bayes model
from sklearn.naive_bayes import BernoulliNB

#Create a Bernoulli Classifier
model = BernoulliNB()

# Train the model using the training sets
model.fit(train_X,train_Y)
#Predict the response for test set

#Import scikit-learn metrics module for accuracy calculation


# Model Accuracy, how often is the classifier correct?




filename = '/content/gdrive/MyDrive/SavedModels/BNB_21_'
filename=filename+".sav"
pickle.dump(model, open(filename, 'wb'))
trainpred = model.predict(train_X)
valpred = model.predict(val_X)
testpred = model.predict(test_X)
train_f1_score = f1_score(train_Y, trainpred, average='weighted')
train_precision_score = precision_score(train_Y, trainpred, average='weighted')
train_recall_score = recall_score(train_Y, trainpred, average='weighted')
train_accuracy_score = accuracy_score(train_Y, trainpred, normalize=True)
print("train_f1_score "+str(train_f1_score) )
print("train_precision_score "+str(train_precision_score))
print("train_recall_score "+str(train_recall_score))
print("train_accuracy_score "+str(train_accuracy_score))
val_f1_score = f1_score(val_Y, valpred, average='weighted')
val_precision_score = precision_score(val_Y, valpred, average='weighted')
val_recall_score = recall_score(val_Y, valpred, average='weighted')
val_accuracy_score = accuracy_score(val_Y, valpred, normalize=True)
print("val_f1_score "+str(val_f1_score) )
print("val_precision_score "+str(val_precision_score))
print("val_recall_score "+str(val_recall_score))
print("val_accuracy_score "+str(val_accuracy_score))

test_f1_score = f1_score(test_Y, testpred, average='weighted')
test_precision_score = precision_score(test_Y, testpred, average='weighted')
test_recall_score = recall_score(test_Y, testpred, average='weighted')
test_accuracy_score = accuracy_score(test_Y, testpred, normalize=True)
print("test_f1_score "+str(test_f1_score) )
print("test_precision_score "+str(test_precision_score))
print("test_recall_score "+str(test_recall_score))
print("test_accuracy_score "+str(test_accuracy_score))<jupyter_output>test_f1_score 0.22155749790124293
test_precision_score 0.21737261595457202
test_recall_score 0.2294981640146879
test_accuracy_score 0.2294981640146879
{'fraction_non_alphanumeric': 0.11496358938219403, 'fraction_numerical': 0.014705191449377496, 'mean_word_length': 11.853864734299517, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '18542', 'license': 'CC0-1.0', 'n_tokens_mistral': 7780, 'n_tokens_neox': 7395, 'n_words': 1158, 'path': 'Classification/NaiveBayes.ipynb', 'repo_name': 'yb24/IMDb-Movie-Rating-Predictor', 'size': '21285'}