<jupyter_start><jupyter_text><jupyter_code># Run this cell to mount your Google Drive.
from google.colab import drive
drive.mount('/content/drive')
!pip install chart_studio
#import libraries
import warnings
warnings.filterwarnings("ignore")
import pandas as pd
import numpy as np
from sklearn.svm import SVC
import matplotlib.pyplot as plt
import chart_studio.plotly as py
import plotly.graph_objs as go
import scipy.stats
from scipy.stats import ttest_ind,mannwhitneyu,wilcoxon
import seaborn as sns
sns.set(style="ticks",color_codes=True)
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import classification_report,confusion_matrix
import gzip
!cp "/content/drive/My Drive/Colab Notebooks/load_data.py" load_data.py
import load_data
!cp "/content/drive/My Drive/Colab Notebooks/data_helper_first.py" data_helper_first.py
import data_helper_first
def parse_gz(path):
    g = gzip.open(path, 'rb')
    for l in g:
        yield eval(l)

def convert_to_DF(path):
    i = 0
    df = {}
    for d in parse_gz(path):
        df[i] = d
        i += 1
    return pd.DataFrame.from_dict(df, orient='index')
# #insert path
path_df1 = '/content/drive/My Drive/Colab Notebooks/Products_review.csv'
path_df2 = '/content/drive/My Drive/Colab Notebooks/Products_reviews.csv'
path_df3 = '/content/drive/My Drive/Colab Notebooks/customer_review.csv'
path_df4 = '/content/drive/My Drive/Colab Notebooks/reviews_Sports_and_Outdoors_5.json.gz'


# Read the data in the CSV file using pandas
df1 = pd.read_csv(path_df1)
#print(df3)
df2 = pd.read_csv(path_df2)

df3 = pd.read_csv(path_df3, encoding='unicode_escape')
#print(df3)

!python load_data.py

df4 = convert_to_DF(path_df4)
# print(df4.columns)
df4 = df4.rename(index=str, columns={"reviewText": "review"})
df4 = df4.rename(index=str, columns={"overall": "rating"})
#print(df4.columns)
#concat the dataframes
df = pd.concat([df1, df2,df3,df4])
#df.drop(columns=['asin', 'asins', 'brand', 'categories', 'date', 'dateAdded','dateUpdated', 'helpful', 'imageURLs', 'keys', 'manufacturer','manufacturerNumber', 'name', 'primaryCategories',  'reviewTime', 'reviewerID', 'reviewerName', 'reviews.date','reviews.dateSeen', 'reviews.didPurchase', 'reviews.doRecommend','reviews.numHelpful', 'reviews.sourceURLs', 'reviews.title','reviews.username', 'sourceURLs', 'summary', 'title', 'unixReviewTime'])

# print(df[:5])
# print(df.columns)

df["label"] = np.where(df.rating >=3, 1, 0)



#print(df['label'])


#Number of the smaller class (negative reviews)
num_to_sample = len(df[df.label == 0])



df_neg = df[df["label"] == 0].sample(n=num_to_sample)


df_pos = df[df["label"] == 1].sample(n=num_to_sample)
df = pd.concat([df_neg, df_pos])


# Get the text from the dataframe
text = df["review"].values
#print(text)

# #clean strings
text= [data_helper_first.clean_str(str(sent)) for sent in text]



# Create labels from the dataframe
labels = df["label"].values


df = pd.concat([pd.DataFrame(text),pd.DataFrame(labels)],axis=1)
#print(len(df)) 144634
#print(df[3000:])
X = text
y = labels

print(df[600:])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)
#change text lower cases and removal of white spaces
lower_text = []
for i in range(0,len(X)):
  s = str(X[i])
  s1 = s.strip()
  lower_text.append(s1.lower())
  #print(“After converting text to lower case\n\n”,lower_text)
import re
#Remove punctuation
punc_text = []
for i in range(0,len(lower_text)):
  s2 = (lower_text[i])
  s3 = re.sub(r'[^\w\s2]',"",s2)
  punc_text.append(s3)
  # print("After removed punctuation\n\n",punc_text[:50])
from sklearn.feature_extraction.text import TfidfVectorizer
#Word vectorization
#Initialize the TF-IDF vectorizer
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5,max_df = 0.7,norm='l2', encoding='latin-1', ngram_range=(1, 2),stop_words='english')
#transform independent variable using TF-IDF vectorizer
print("\n")
X_tfidf = tfidf.fit_transform(punc_text)
print("After vectorized text data\n\n",X_tfidf)
#Split the data into train and testing
X_train, X_test, Y_train, Y_test = train_test_split(X_tfidf, y, test_size=0.3, random_state = 42)
#Print training data
print("\n")
print("Training data\n\n",X_train,"\n",Y_train)
print("\n\n")
#Print testing data
print("Testing data\n\n",X_test)
print("\n\n")
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
#Build the Naive_bayes model
clf = MultinomialNB()
#Fit train and test into the model
clf.fit(X_train, Y_train)
#Predict the result
y_pred = clf.predict(X_test)
# print(y_pred[:100])
#classification report & confusion matrix
print("Confusion Matrix\n",confusion_matrix(Y_test,y_pred))
print("\n")
print("Classification Report\n",classification_report(Y_test,y_pred))
print("\n")
print("Accuracy : ",accuracy_score(Y_test,y_pred)*100)
import pickle

with open('/content/drive/My Drive/Colab Notebooks/training_models/mnb_trained_model.pkl', 'wb') as f:
    pickle.dump(clf, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1427157561361837, 'fraction_numerical': 0.015637371338083926, 'mean_word_length': 6.147100424328147, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '234112', 'license': 'Apache-2.0', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1851, 'n_words': 415, 'path': 'Multinomial_NB.ipynb', 'repo_name': 'oguchi-ebube/Sugar-Instance-Packager', 'size': '5052'}