<jupyter_start><jupyter_text>1) Importing the datasets<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import spacy
nlp = spacy.load('en_core_web_sm')
data_yelp = pd.read_csv('yelp_labelled.txt', sep='\t',header=None)
data_yelp.head()
# review and sentiment
# 0-Negative, 1-Positive for positive review
# Assign column names
column_name = ['Review', 'Sentiment']
data_yelp.columns = column_name
data_yelp.head()
data_yelp.shape
# 1000 rows (reviews), 2 columns (Sentiments)
data_amazon = pd.read_csv('amazon_cells_labelled.txt',sep='\t',header=None)
data_amazon.head()
# review and sentiment
# 0-Negative, 1-Positive for positive review
data_amazon.columns = column_name
data_amazon.head()
data_amazon.shape
data_imdb = pd.read_csv('imdb_labelled.txt',sep='\t',header=None)
data_imdb.head()
data_imdb.columns = columan_name
data_imdb.head()
data_imdb.shape
# Append all the data in a single dataframe
data = data_yelp.append([data_amazon, data_imdb],ignore_index=True)
data.shape
data.head()
# check distribution of sentiments
data['Sentiment'].value_counts()

# 1346 positive reviews
# 1362 Negative reviews
# check for null values
data.isnull().sum()

# no null values in the data
x = data['Review']
y = data['Sentiment']<jupyter_output><empty_output><jupyter_text>2) Data Cleaning<jupyter_code># here we will remove stopwords, punctuations
# as well as we will apply lemmatization<jupyter_output><empty_output><jupyter_text>Create a function to clean the data<jupyter_code>import string
punct = string.punctuation
punct
from spacy.lang.en.stop_words import STOP_WORDS
stopwords = list(STOP_WORDS) # list of stopwords
# creating a function for data cleaning
def text_data_cleaning(sentence):
  doc = nlp(sentence)

  tokens = [] # list of tokens
  for token in doc:
    if token.lemma_ != "-PRON-":
      temp = token.lemma_.lower().strip()
    else:
      temp = token.lower_
    tokens.append(temp)
 
  cleaned_tokens = []
  for token in tokens:
    if token not in stopwords and token not in punct:
      cleaned_tokens.append(token)
  return cleaned_tokens
# if root form of that word is not pronoun then it is going to convert that into lower form
# and if that word is a proper noun, then we are directly taking lower form, because there is no lemma for proper noun
text_data_cleaning("Hello all, It's a beautiful day outside there!")
# stopwords and punctuations removed<jupyter_output><empty_output><jupyter_text>Vectorization Feature Engineering (TF-IDF)<jupyter_code>from sklearn.svm import LinearSVC
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
tfidf = TfidfVectorizer(tokenizer=text_data_cleaning)
# tokenizer=text_data_cleaning, tokenization will be done according to this function
classifier = LinearSVC()<jupyter_output><empty_output><jupyter_text>3) Train the model  Splitting the dataset into the Train and Test set<jupyter_code>from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)
x_train.shape, x_test.shape
# 2198 samples in training dataset and 550 in test dataset
x_train.head()<jupyter_output><empty_output><jupyter_text>Fit the x_train and y_train<jupyter_code>clf = Pipeline([('tfidf',tfidf), ('clf',classifier)])
# it will first do vectorization and then it will do classification
clf.fit(x_train, y_train)
# in this we don't need to prepare the dataset for testing(x_test)<jupyter_output><empty_output><jupyter_text>4) Predict the Test set results<jupyter_code>from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
y_pred = clf.predict(x_test)
# confusion_matrix
confusion_matrix(y_test, y_pred)
# classification_report
print(classification_report(y_test, y_pred))
# we are getting almost 77% accuracy
accuracy_score(y_test, y_pred)
# 76% accuracy
clf.predict(["Wow, I am learning Natural Language Processing in fun fashion!"])
# output is 1, that means review is positive
clf.predict(["It's hard to learn new things!"])
# output is 0, that means review is Negative<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09163442940038685, 'fraction_numerical': 0.008945841392649903, 'mean_word_length': 6.309187279151944, 'pattern_counts': {'":': 1, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '127980', 'license': 'MIT', 'n_tokens_mistral': 1407, 'n_tokens_neox': 1341, 'n_words': 396, 'path': '06IMDBAndAmazonReviewClassificationWithSpaCy.ipynb', 'repo_name': 'MBadriNarayanan/NaturalLanguageProcessing', 'size': '4136'}