<jupyter_start><jupyter_text>Sentiment Classifier using Bag of Words (BoW) techniques<jupyter_code>import json
# Import pandas
import pandas as pd
# Import CountVectorizer from scikit learn
from sklearn.feature_extraction.text import CountVectorizer
COLUMNS = ['reviews', 'sentiment']

test_df = pd.read_json('movie_reviews_test.json')
test_df.columns = COLUMNS

train_df = pd.read_json('movie_reviews_train.json')
train_df.columns = COLUMNS
max_colwidth_ref = pd.get_option('display.max_colwidth')

# Allow to display full column length
pd.set_option('display.max_colwidth', -1)
# Print data frames before cleaning
print('Training materials')
print(train_df['reviews'].shape)
print(train_df['reviews'].head(10))

print('Testing materials')
print(test_df['reviews'].shape)
print(test_df['reviews'].head(25))
# Cleaning function (removes some html tags)
def clean_br(review):
  for html_tag in ('<br>', '<br/>', '<br >', '<br />'):
    review = review.replace(html_tag, '')
  return review
%time X_test = test_df['reviews'].apply(clean_br)
%time X_train = train_df['reviews'].apply(clean_br)
print('Training materials')
print(X_train.shape)
print(X_train.head(10))
print('Testing materials')
print(X_test.shape)
print(X_test.head(10))
# Setting default value back to normal
pd.set_option('display.max_colwidth', max_colwidth_ref)
# Create a CountVectorizer instance
vectorizer = CountVectorizer(stop_words='english')
# N-grams
# vectorizer = CountVectorizer(stop_words='english', ngram_range=(1, 1))
# vectorizer = CountVectorizer(stop_words='english', ngram_range=(1, 2))
# vectorizer = CountVectorizer(stop_words='english', ngram_range=(1, 3))

# Fit and transform training materials
X_train_bow = vectorizer.fit_transform(X_train)

# Transform testing materials
X_test_bow = vectorizer.transform(X_test)
print('test', X_test_bow.shape)
print('train', X_train_bow.shape)
from sklearn.naive_bayes import MultinomialNB
y_test = test_df['sentiment']
y_train = train_df['sentiment']
# Sentiment map
SENTIMENTS = {
    0: 'Negative',
    1: 'Positive'
}
# Create a MultinomialNB instance
clf = MultinomialNB()

# Fit the classifier
clf.fit(X_train_bow, y_train)

# Measure accuracy of the trained model
accuracy = clf.score(X_test_bow, y_test)
print("The classifier accuracy on the test set is {0:.2f}%".format(accuracy * 100))
# Predict the sentiment of some reviews
for (review, expected) in (
    ("The movie was awful. The music was hard to ear and the acting bad.", 0),
    ("The movie was great! The music was overwhelming. Really nice!", 1),
    ("The movie was not so good, even pretty bad :( Actors were not brillant.", 0),
):
  prediction = clf.predict(vectorizer.transform([review]))[0]
  print(
      "The sentiment predicted by the classifier is {} (expected {})"
      .format(SENTIMENTS[prediction], SENTIMENTS[expected])
  )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12770411723656663, 'fraction_numerical': 0.009071877180739707, 'mean_word_length': 6.313775510204081, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '643325', 'license': 'MIT', 'n_tokens_mistral': 986, 'n_tokens_neox': 943, 'n_words': 243, 'path': 'bag_of_words_demonstration.ipynb', 'repo_name': 'nam4dev/nlp_demonstration', 'size': '2866'}