<jupyter_start><jupyter_text><jupyter_code>import json
file_name = open('Books_small_10000.json')
reviews = []
import random

class Sentiment:
  NEGATIVE = "NEGATIVE"
  NEUTRAL = "NEUTRAL"
  POSITIVE = "POSITIVE"

class Review:
  def __init__(self,text,score):
    self.text = text
    self.score = score
    self.sentiment = self.get_sentiment()

  def get_sentiment(self):
    if self.score <= 2:
      return Sentiment.NEGATIVE
    elif self.score == 3:
      return Sentiment.NEUTRAL
    else:
      return Sentiment.POSITIVE
    
class ReviewContainer:
  def __init__(self,reviews):
    self.reviews = reviews

  def get_text(self):
    return [x.text for x in self.reviews]

  def get_sentiment(self):
    return [x.sentiment for x in self.reviews]


  def evenly_distribute(self):
      negative = list(filter(lambda x:x.sentiment == Sentiment.NEGATIVE, self.reviews))
      positive = list(filter(lambda x:x.sentiment == Sentiment.POSITIVE,self.reviews))
      neutral = list(filter(lambda x:x.sentiment == Sentiment.NEUTRAL,self.reviews))
      positive_shrunk = positive[:len(negative)]
      self.reviews = negative + positive_shrunk
      random.shuffle(self.reviews)
import json

for line in file_name:
  review = json.loads(line)
  print (review['reviewText'])
  print (review['overall'])
  reviews.append(Review(review['reviewText'],review['overall']))
  
reviews[5].text
from sklearn.model_selection import train_test_split

training , test = train_test_split(reviews , test_size = 0.33 , random_state = 42)

train_container = ReviewContainer(training)
test_container = ReviewContainer(test)

cont.evenly_distribute()
len(cont.reviews)

train_container.evenly_distribute()
train_x = train_container.get_text()
train_y = train_container.get_sentiment()

test_container.evenly_distribute()
test_x = test_container.get_text()
test_y = test_container.get_sentiment()

print (train_y.count(Sentiment.POSITIVE))
print(train_y.count(Sentiment.NEGATIVE))



from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

vectorizer = TfidfVectorizer()
train_x_vectors= vectorizer.fit_transform(train_x)
test_x_vectors = vectorizer.transform(test_x)
print (train_x[0])
print (train_x_vectors[0].toarray())

from sklearn import svm

clf_svm = svm.SVC(kernel='linear')
clf_svm.fit(train_x_vectors,train_y)

test_x[0]

clf_svm.predict(test_x_vectors[0])
from sklearn.tree import DecisionTreeClassifier
clf_dec = DecisionTreeClassifier()
clf_dec.fit(train_x_vectors,train_y)
clf_dec.predict(test_x_vectors[0])
from sklearn.naive_bayes import GaussianNB

clf_gnb = GaussianNB()
clf_gnb.fit(train_x_vectors,train_y)
clf.gnb.predict(test_x_vectors[0])
from sklearn.linear_model import LogisticRegression

clf_log = LogisticRegression()
clf_log.fit(train_x_vectors,train_y)
clf_log.predict(test_x_vectors)
# Mean Accuracy

print(clf_svm.score(train_x_vectors,train_y))
print(clf_dec.score(train_x_vectors,train_y))
print(clf_log.score(train_x_vectors,train_y))
from sklearn.metrics import f1_score

f1_score(test_y,clf_svm.predict(test_x_vectors),average = None, labels=[Sentiment.POSITIVE,Sentiment.NEUTRAL,Sentiment.NEGATIVE])
#f1_score(test_y,clf_log.predict(test_x_vectors),average = None, labels=[Sentiment.POSITIVE,Sentiment.NEUTRAL,Sentiment.NEGATIVE])
#f1_score(test_y,clf_dec.predict(test_x_vectors),average = None, labels=[Sentiment.POSITIVE,Sentiment.NEUTRAL,Sentiment.NEGATIVE])
test_y.count(Sentiment.NEGATIVE)
test_set = ['Book is Fun' , 'Hate the Concept' , 'Author is best']
new_test = vectorizer.transform(test_set)
clf_svm.predict(new_test)
from sklearn.model_selection import GridSearchCV

parameters = {'kernel' : ('linear','rbf'),'C' : (1,4,8,16,32)}
svc = svm.SVC()
clf = GridSearchCV(svc , parameters , cv =5)
clf.fit(train_x_vectors,train_y)
print(clf.score(test_x_vectors,train_y))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1148051948051948, 'fraction_numerical': 0.008051948051948052, 'mean_word_length': 6.463178294573644, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '653502', 'license': 'MIT', 'n_tokens_mistral': 1481, 'n_tokens_neox': 1406, 'n_words': 224, 'path': 'SkLearn_using_Models_Prediction.ipynb', 'repo_name': 'manojgupta3051994/dsmp-pre-work', 'size': '3850'}