<jupyter_start><jupyter_text>In this Notebook following steps were followed - 1. Tokenised Text - Text was tokeinsed2. Stemmed Text - Tokenised text was stemmed 3. Stopwords Deleted - stopwords were removed from Stemmed Text4. Vectorized Form - After all preprocessing, text was finally transformed into vectorized form5. Oversampled Data - Data was oversampled with help of ADASYN6. Train-Test Split - Oversampled data was split in 70-30 ratio7. After this model was trained with (X_train, y_train) and tested on (X_test,y_test) 8. Model and vectorizer were saved on disk in form of .pkl file<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!cd '/content/drive/My Drive/ML Projects/MusicSentimentAnalysis/'
# Importing necessary paackages
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns

import re
import nltk
# from nltk.stem.porter import PorterStemmer
# from nltk.tokenize import word_tokenize, sent_tokenize
# from nltk.stem.wordnet import WordNetLemmatizer
import string

from imblearn.over_sampling import ADASYN

from sklearn.model_selection import train_test_split, cross_val_score 
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, plot_confusion_matrix
from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer 
from sklearn.metrics import precision_recall_fscore_support as score, precision_score, recall_score, f1_score
from sklearn.svm import SVC
seed = 42

from sklearn.externals import joblib
import nltk
nltk.download("stopwords")
nltk.download("punkt")
nltk.download("wordnet")
data = pd.read_csv('/content/drive/My Drive/ML Projects/MusicSentimentAnalysis/Musical_instruments_reviews.csv')   # Load Data
data.head()
# Function for cleaning dataset

class Sentiment:
    POSITIVE = "POSITIVE"
    NEGATIVE = "NEGATIVE"
    NEUTRAL = "NEUTRAL"

#Creating Columns for the year, month and day
def get_year(reviewTime):
    return reviewTime.split(",")[1].strip(" ")

#Function to create 
def get_day(reviewTime):
    return reviewTime.split(" ")[1].strip(",")

#Sentiment Score     (Categorising Opinions Task-1)  
def get_sentiment(score):
    if score <= 2:
        return 'NEGATIVE'
    elif score == 3:
        return 'NEUTRAL'
    else: 
        return 'POSITIVE'
    
def text_clean(text):
    text = str(text).lower()
    text = re.sub(r'@[A-Za-z0-9]','',text)# Removes @mentions
    text = re.sub(r'#','',text)# Removing the Hashtag
    text = re.sub(r'RT[\s]','',text)# Removing RT
    text = re.sub(r'_','',text)# Removing RT
    return text
# Cleaning The dataset

data['count'] = 1 
data['year'] = data.reviewTime.apply(get_year)
data['month'] =  data['reviewTime'].str[:2]
data['day'] = data.reviewTime.apply(get_day)
data['Sentiment'] = data.overall.apply(get_sentiment)

#Converting Column data type from object to integer
data['year'] = data['year'].astype('int64')
data['month'] = data['month'].astype('int64')
data['day'] = data['day'].astype('int64')

#Cleaning Text
data['reviewText'] = data['reviewText'].apply(text_clean)
X = data['reviewText']    # Initialising Features
y = data['Sentiment']     # Initialising Target
vectorizer = TfidfVectorizer(stop_words='english')        # Initialising Vectorizer
X = vectorizer.fit_transform(X)                           # Input Data Vectorized
joblib_file = "/content/drive/My Drive/ML Projects/MusicSentimentAnalysis/vect.pkl"    # Saving Vectorizer on disk
joblib.dump(vectorizer, joblib_file)
smote = ADASYN(random_state= seed)      # Over-Sampling Data
X_smote, y_smote = smote.fit_resample(X,y)
X_train, X_test, y_train, y_test = train_test_split(X_smote, y_smote, test_size= 0.3, random_state= seed)     # Train-Test Data Split
svm = SVC(kernel='rbf')     # Initialising Model
svm.fit(X_train,y_train)     # Training Model
svmc = svm.predict(X_test)    # Testing Model<jupyter_output><empty_output><jupyter_text>Validation Metrics  Accuracy, Precision, Recall, F1_Score<jupyter_code># accuracy: 
acc = accuracy_score(y_test, svmc)
print('Accuracy: %f' % acc)

# precision: tp / (tp + fp)
precision = precision_score(y_test, svmc,average='weighted')
print('Precision: %f' % precision)

# recall: tp / (tp + fn)
recall = recall_score(y_test, svmc,average='weighted')
print('Recall: %f' % recall)

# f1: tp / (tp + fp + fn)
f1 = f1_score(y_test, svmc,average='weighted')
print('F1 score: %f' % f1)<jupyter_output>Accuracy: 0.991934
Precision: 0.992117
Recall: 0.991934
F1 score: 0.991947<jupyter_text>Classification Report<jupyter_code>print('Support Vector Classifier')
plot_confusion_matrix(svm, X_test, y_test)  
plt.show()
print()
print('Classification Report')
print(classification_report(y_test, svmc))
print('Model Accuracy: ',accuracy_score(y_test,svmc))
#  save the model to disk
joblib_file = "/content/drive/My Drive/ML Projects/MusicSentimentAnalysis/SVMModel.pkl"  
joblib.dump(svm, joblib_file)<jupyter_output><empty_output><jupyter_text>Cross Validation with 5 iter<jupyter_code>score = cross_val_score(svm, X_smote, y_smote, scoring='accuracy')
print("%f mean accuracy with a standard deviation of %f" % (score.mean(), score.std()))<jupyter_output>0.981925 mean accuracy with a standard deviation of 0.032491<jupyter_text>Exploratory Data Analysis  From graph given below we can conclude that most of product got 5 star rating by people proving the quality of products<jupyter_code>#Product Valuation
sns.countplot(data=data, x='overall')
plt.ylabel('Overall Count')
plt.title('PRODUCT OVERALL')
plt.grid()
plt.show()<jupyter_output><empty_output><jupyter_text>From graph given below this we can conclude that most rating given by customers are positive and rarely negative or neutral.Hence we can also conclude that given dataset is unbalanced<jupyter_code>#Reviews Sentiment
size_complete = data['Sentiment'].value_counts()
colors_complete = ['lightgreen', 'palegoldenrod','salmon']
labels_complete = "Positive", "Neutral","Negative"
explode = [0, 0.01,0.01]

#(0,0) is to create the circle and 0.5 for the width of the circle, white is for the center of the circle
my_circle_complete = plt.Circle((0, 0), 0.45, color = 'white')

plt.figure(figsize=(8,6))
plt.pie(size_complete, colors = colors_complete, labels = labels_complete, shadow = False, explode = explode, autopct = '%.2f%%',textprops=dict(color='blue'))
plt.title('PROPORTIONS BY SENTIMENT', fontsize = 15)
p = plt.gcf()
p.gca().add_artist(my_circle_complete)
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>From the graph given below we can see that which customer has given us good business<jupyter_code>#Top 10 reviewers
top_ten = data['reviewerName'].value_counts()
top_ten = top_ten.head(10)
top_ten = pd.DataFrame(top_ten)
top_ten.columns = ['NumberOfdata']

top_ten.plot(color='darkred',marker='v',linestyle='-')
plt.xticks(rotation=90)
plt.title('TOP 10 REVIEWERS')
plt.grid()
plt.xlabel('Reviewers')
plt.ylabel('Nº of Reviews')
plt.show()<jupyter_output><empty_output><jupyter_text>From graph given below we can find out about our growth of business yearly<jupyter_code>#Grouping By  year
year_review = data.groupby(by='year').sum()
year_review = year_review.drop(columns=['month','day','overall','unixReviewTime'])
year_review.columns = ['trend']

#Creating index
#Creating index for the plot
year_index = year_review.index.unique()

#Plot
plt.figure(figsize=(14,10))
year_review.plot(color='magenta',linestyle='-',marker='o')
plt.xticks(year_index,rotation=90)
plt.grid()
plt.ylabel('Reviews Count')
plt.title('REVIEWS PER YEAR TREND')
plt.show()<jupyter_output><empty_output><jupyter_text>From graph given below we can find out about our growth of business monthly<jupyter_code># Grouping by month
month_review = data.groupby(by='month').sum()
month_review = month_review.drop(columns=['year','day','overall','unixReviewTime'])
month_review.columns = ['trend']

#Creating index for the plot
month_index = month_review.index.unique()

#Creating plot
plt.figure(figsize=(14,10))
month_review.plot(color='orange',linestyle='-',marker='o')
plt.xticks(month_index,rotation=90)
plt.grid()
plt.ylabel('Reviews Count')
plt.title('REVIEWS PER MONTH TREND')
plt.show()<jupyter_output><empty_output><jupyter_text>From graph given below we can find out about our growth of business daily<jupyter_code># Trend by day of the month
day = data
day = day.sort_values(by=['day'],ascending=True)
day =  day.groupby(by=['day']).sum()
day = day.drop(columns=['overall', 'unixReviewTime','year','month'])
day.columns = ['trend']

# Line plot
plt.figure(figsize=(10,8))
sns.lineplot(data=day, x=day.index, y='trend',color='blue')
plt.xticks(day.index)
plt.grid()
plt.ylabel('Review Count')
plt.xlabel('Days of the Month')
plt.title('REVIEW TREND PER DAYS OF THE MONTHS')
plt.show()<jupyter_output><empty_output><jupyter_text>From graph given below we can find about approximately how long reviews are generally<jupyter_code># Creating a column to see the len of each message
data['Len'] = data['reviewText'].apply(lambda x: len(x.split(' ')))

# Senteces Distribution
unique = data['Len'].unique()
plt.figure(figsize=(20,8))
sns.distplot(data['Len'],color='green')
plt.title('MESSEAGES LEN DISTRIBUTION')
plt.xlabel('Number of words per message')
plt.grid()
plt.xticks(rotation=90)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12853470437017994, 'fraction_numerical': 0.014138817480719794, 'mean_word_length': 6.176787086856264, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '336158', 'license': 'MIT', 'n_tokens_mistral': 3351, 'n_tokens_neox': 3171, 'n_words': 793, 'path': 'Projects/Musical_InstrumentReview/MusicalReview.ipynb', 'repo_name': 'Kartik192192/Hacktoberfest2021-9', 'size': '9336'}