<jupyter_start><jupyter_text>Loading Dataset<jupyter_code>working_dir_path = 'Reviews.csv'
df = pd.read_csv(working_dir_path,encoding = 'latin')
df.head()<jupyter_output><empty_output><jupyter_text>Assessing Dataset<jupyter_code>df.shape
df.columns<jupyter_output><empty_output><jupyter_text>Make sentiments positive that has score 4 or higher<jupyter_code>df.loc[(df.Score >= 4),'Sentiment']='Positive'<jupyter_output><empty_output><jupyter_text>Make sentiments neutral that has score 3<jupyter_code>df.loc[(df.Score == 3),'Sentiment']='Neutral'<jupyter_output><empty_output><jupyter_text>Make sentiments negative that has score less than or equals to 2<jupyter_code>df.loc[(df.Score <= 2),'Sentiment']='Negative'<jupyter_output><empty_output><jupyter_text>Visualize labels, Sentiment<jupyter_code>sns.set(font_scale=1)
sns.catplot("Sentiment", data=df, kind="count", height=5)
df['Sentiment'].value_counts()<jupyter_output><empty_output><jupyter_text>A) Removing @user, HTTP and URLs  Preprocessing<jupyter_code>df['Text'] = df['Text'].str.lower()<jupyter_output><empty_output><jupyter_text>A) Removing Punctuations, Numbers, and Special Characters<jupyter_code># remove special characters, numbers, punctuations
df['Text'] = df['Text'].str.replace('[^a-zA-Z#]+',' ')<jupyter_output><empty_output><jupyter_text>B) Removing Short Words<jupyter_code># remove short words
df['Text'] = df['Text'].apply(lambda x: ' '.join([w for w in x.split() if len(w) > 2]))<jupyter_output><empty_output><jupyter_text>C) Tokenization<jupyter_code># create new variable tokenized texts
tokenized_texts = df['Text'].apply(lambda x: x.split())<jupyter_output><empty_output><jupyter_text>D) Stemming<jupyter_code>stemmer = PorterStemmer()

# apply stemmer for tokenized_tweet
tokenized_texts = tokenized_texts.apply(lambda x: [stemmer.stem(i) for i in x])

# join tokens into one sentence
for i in range(len(tokenized_texts)):
    tokenized_texts[i] = ' '.join(tokenized_texts[i])
# change df['Texts'] to tokenized_texts

df['Text']  = tokenized_texts
df.head(5)<jupyter_output><empty_output><jupyter_text>Extracting Features from cleaned dataset<jupyter_code>new_df = df[['Text','Sentiment']]<jupyter_output><empty_output><jupyter_text>Removing Stopwords<jupyter_code>nltk.download('stopwords')
from nltk.corpus import stopwords
stop = stopwords.words('english')
new_df['Text'].apply(lambda x: [item for item in x if item not in stop])<jupyter_output><empty_output><jupyter_text>Splitting the Dataset into Training And Testing Dataset ( For Multiclass Classification)<jupyter_code>train,test = train_test_split(new_df,test_size = 0.2,random_state=0,stratify = new_df.Sentiment.values) #stratification means that the train_test_split method returns training and test subsets that have the same proportions of class labels as the input dataset.
print("Train shape : ", train.shape)
print("Test shape : ", test.shape)
from nltk.corpus import stopwords
stop = list(stopwords.words('english'))
vectorizer = CountVectorizer(decode_error = 'replace',stop_words = stop)

X_train = vectorizer.fit_transform(train.Text.values)
X_test = vectorizer.transform(test.Text.values)

y_train = train.Sentiment.values
y_test = test.Sentiment.values

print("X_train.shape : ", X_train.shape)
print("X_test.shape : ", X_test.shape)
print("y_train.shape : ", y_train.shape)
print("y_test.shape : ", y_test.shape)<jupyter_output>X_train.shape :  (454763, 72127)
X_test.shape :  (113691, 72127)
y_train.shape :  (454763,)
y_test.shape :  (113691,)<jupyter_text>MULTICLASS Classification 1) Naive Bayes Classifier<jupyter_code>from sklearn.naive_bayes import MultinomialNB

naiveByes_clf = MultinomialNB()

naiveByes_clf.fit(X_train,y_train)

NB_prediction = naiveByes_clf.predict(X_test)
NB_accuracy = accuracy_score(y_test,NB_prediction)
print("Training accuracy Score    : ", naiveByes_clf.score(X_train,y_train))
print("Testing accuracy Score : ", NB_accuracy )
print(classification_report(NB_prediction,y_test))<jupyter_output>Training accuracy Score    :  0.8339134890041626
Testing accuracy Score :  0.8280602686228461
              precision    recall  f1-score   support

    Negative       0.64      0.64      0.64     16614
     Neutral       0.32      0.33      0.32      8240
    Positive       0.91      0.91      0.91     88837

    accuracy                           0.83    113691
   macro avg       0.62      0.63      0.63    113691
weighted avg       0.83      0.83      0.83    113691<jupyter_text>2) Stochastic Gradient Descent-SGD Classifier<jupyter_code>from sklearn.linear_model import SGDClassifier

sgd_clf = SGDClassifier(loss = 'hinge', penalty = 'l2', random_state=0)

sgd_clf.fit(X_train,y_train)

sgd_prediction = sgd_clf.predict(X_test)
sgd_accuracy = accuracy_score(y_test,sgd_prediction)
print("Training accuracy Score    : ",sgd_clf.score(X_train,y_train))
print("Testing accuracy Score : ",sgd_accuracy )
print(classification_report(sgd_prediction,y_test))<jupyter_output>Training accuracy Score    :  0.8671066027799096
Testing accuracy Score :  0.8632257610540852
              precision    recall  f1-score   support

    Negative       0.64      0.76      0.69     13866
     Neutral       0.06      0.67      0.12       815
    Positive       0.98      0.88      0.93     99010

    accuracy                           0.86    113691
   macro avg       0.56      0.77      0.58    113691
weighted avg       0.93      0.86      0.89    113691<jupyter_text>3) Random Forest Classifier<jupyter_code>rf_clf = RandomForestClassifier()

rf_clf.fit(X_train,y_train)

rf_prediction = rf_clf.predict(X_test)
rf_accuracy = accuracy_score(y_test,rf_prediction)
print("Training accuracy Score    : ",rf_clf.score(X_train,y_train))
print("Testing accuracy Score : ",rf_accuracy )
print(classification_report(rf_prediction,y_test))<jupyter_output>Training accuracy Score    :  0.9998790578828972
Testing accuracy Score :  0.8856989559419831
              precision    recall  f1-score   support

    Negative       0.53      0.97      0.69      9029
     Neutral       0.39      1.00      0.56      3308
    Positive       1.00      0.87      0.93    101354

    accuracy                           0.89    113691
   macro avg       0.64      0.95      0.73    113691
weighted avg       0.94      0.89      0.90    113691<jupyter_text>4) Logistic Regression<jupyter_code>logreg = LogisticRegression()

logreg.fit(X_train, y_train)

logreg_prediction = logreg.predict(X_test)
logreg_accuracy = accuracy_score(y_test,logreg_prediction)
print("Training accuracy Score    : ",logreg.score(X_train,y_train))
print("Testing accuracy Score : ",logreg_accuracy )
print(classification_report(logreg_prediction,y_test))<jupyter_output>Training accuracy Score    :  0.8765686742325123
Testing accuracy Score :  0.8673861607339192
              precision    recall  f1-score   support

    Negative       0.67      0.76      0.71     14543
     Neutral       0.21      0.51      0.30      3593
    Positive       0.97      0.90      0.93     95555

    accuracy                           0.87    113691
   macro avg       0.62      0.72      0.65    113691
weighted avg       0.90      0.87      0.88    113691<jupyter_text>Test Accuracies<jupyter_code>models = pd.DataFrame({
    'Model': ['Logistic Regression', 
              'Random Forest', 'Naive Bayes', 
              'Stochastic Gradient Decent'],
    'Test accuracy': [logreg_accuracy, 
              rf_accuracy, NB_accuracy, 
              sgd_accuracy]})

models.sort_values(by='Test accuracy', ascending=False)<jupyter_output><empty_output><jupyter_text>Understanding the emotions of people. a) What are they feeling negative about?<jupyter_code># import numpy as npy
# from PIL import Image

from wordcloud import WordCloud
# create text from just normal tweets
normal_words = ' '.join([text for text in new_df['Text'][new_df['Sentiment'] == 'Negative']])

# mask_virus = npy.array(Image.open('/content/drive/MyDrive/NLP Group Project/Mask.png'))
wordcloud = WordCloud(width=800, height=500, background_color = 'white', random_state=21, max_font_size=110).generate(normal_words)
plt.figure(figsize=(10, 7))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>b) What are they feeling positive about?<jupyter_code># create text from just normal tweets
normal_words = ' '.join([text for text in new_df['Text'][new_df['Sentiment'] == 'Positive']])

# mask_virus = npy.array(Image.open('/content/drive/MyDrive/NLP Group Project/Mask.png'))
wordcloud = WordCloud(width=800, height=500, background_color = 'white', random_state=21, max_font_size=110).generate(normal_words)
plt.figure(figsize=(10, 7))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>Chart for negative words<jupyter_code>neg_words = ' '.join([text for text in new_df['Text'][new_df['Sentiment'] == 'Negative']]).split()

neg_words_freq = {}
stop = list(stopwords.words('english'))


for word in neg_words:
  if word not in stop:
    if word not in neg_words_freq:
      neg_words_freq[word] = 1
    else:
      neg_words_freq[word] += 1


neg_words_freq = sorted(neg_words_freq.items(), key=lambda neg_words_freq: neg_words_freq[1], reverse=True)


neg_words_freq = neg_words_freq[:20]

plt.rcParams["figure.figsize"] = (20,5)
fig = plt.figure()
ax = fig.add_axes([0,0,1,1])
freqs = []
values = []

for idx,value in enumerate(neg_words_freq):
  freqs.append(value[0])
  values.append(value[1])

ax.bar(freqs,values)
plt.show()<jupyter_output><empty_output><jupyter_text>Chart for positive words<jupyter_code>pos_words = ' '.join([text for text in new_df['Text'][new_df['Sentiment'] == 'Positive']]).split()

pos_words_freq = {}
stop = list(stopwords.words('english'))

for word in pos_words:
  if word not in stop:
    if word not in pos_words_freq:
      pos_words_freq[word] = 1
    else:
      pos_words_freq[word] += 1


pos_words_freq = sorted(pos_words_freq.items(), key=lambda pos_words_freq: pos_words_freq[1], reverse=True)


pos_words_freq = pos_words_freq[:20]

plt.rcParams["figure.figsize"] = (20,5)
fig = plt.figure()
ax = fig.add_axes([0,0,1,1])
freqs = []
values = []

for idx,value in enumerate(pos_words_freq):
  freqs.append(value[0])
  values.append(value[1])

ax.bar(freqs,values)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11731251204935415, 'fraction_numerical': 0.05475226527858107, 'mean_word_length': 4.392411642411642, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '254999', 'license': 'MIT', 'n_tokens_mistral': 4035, 'n_tokens_neox': 3664, 'n_words': 696, 'path': 'Sentiment Analysis.ipynb', 'repo_name': 'rafin007/rm-sentiment-analysis', 'size': '10374'}