<jupyter_start><jupyter_text>Sentiment Analysis - DecisionTree, RandomForest & SVC Import Library and Load Data<jupyter_code>import pandas as pd
import collections
import json
import matplotlib.pyplot as plt
import seaborn as sns
#import wordcloud
import re
import nltk
#import langdetect 
! pip install -U textblob
! python -m textblob.download_corpora lite
from textblob import TextBlob
from sklearn import feature_extraction, manifold
from sklearn.dummy import DummyClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix
import numpy as np
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score
#import gensim.downloader as gensim_api
#import gensim
from sklearn.model_selection import train_test_split
! pip install langdetect
from langdetect import detect
RandomState = 233
!pip3 install pickle5
import pickle5 as pickle

with open("Data/sg_reviews.pkl", "rb") as fh:
    raw_df = pickle.load(fh) 
#Uncomment for Australia market data
'''
with open("Data/au_reviews.pkl", "rb") as fh:
    raw_df = pickle.load(fh) 
    '''
raw_df.head()
raw_df.dropna(subset=['review', 'rating'],inplace=True)

raw_df = raw_df[['review','rating']]
raw_df.loc[raw_df['rating']>2,'label'] = 'positive' 
raw_df.loc[raw_df['rating']<3,'label'] = 'negative'<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code># check if there is any alphabetics in a string

def check_letter(string):
    for c in string:
        if ((c > 'A') and (c < 'z')):
            return True
    return False
# Alternative language detector --- based on UTF-8, if contain any character outside this range, considered as foreign language
def check_non_eng(string):
    for char in string:
        if char < '~':
            return False
    return True
        

#Remove reviews that doesnot contain alphabetic letter for langdetect module
'''
raw_df['letter'] = raw_df['review'].apply(lambda x: check_letter(x))
            
# Filter out reviews that contains only emoji (no letter)
#print(raw_df[raw_df.letter == False])

raw_df = raw_df[raw_df.letter == True]
raw_df['lang'] = raw_df['review'].apply(detect)
raw_df[raw_df.lang != 'en']


raw_df_en = raw_df[raw_df.lang == 'en']

#raw_df.head(10)
'''
# Lemmatization
def preprocess_text(text,  flg_lemm=True):
    ## clean (convert to lowercase and remove punctuations and characters and then strip)
    text = re.sub(r'[^\w\s]', '', str(text).lower().strip())
            
    ## Tokenize (convert from string to list)
    lst_text = text.split()
                
    ## Lemmatisation (convert the word into root word)
    if flg_lemm == True:
        lem = nltk.stem.wordnet.WordNetLemmatizer()
        lst_text = [lem.lemmatize(word) for word in lst_text]
            
    ## back to string from list
    text = " ".join(lst_text)
    return text<jupyter_output><empty_output><jupyter_text>Train test split<jupyter_code># Split train test set

X_train, X_test, y_train, y_test = train_test_split(raw_df.review, raw_df.label, test_size=0.2, random_state=RandomState)<jupyter_output><empty_output><jupyter_text>Vectorization<jupyter_code># vectorize datasets using Word2vec -- TFIDF
  # input processed datasets and return vectorized datasets
def get_vect(X_train, X_test, StopWords='english',MinDf=500,Ngram =(1,2), add_ft=False):
    
    vectorizer = TfidfVectorizer(stop_words=StopWords,min_df=MinDf,ngram_range=Ngram)
    X_train_vectorized = vectorizer.fit_transform(X_train)
    X_test_vectorized = vectorizer.transform(X_test)
    
    # Adding Polariry as feature (using TextBlob)
    if add_ft == True:
        X_pol_train = X_train.apply(lambda x: TextBlob(x).sentiment.polarity)
        X_pol_test = X_test.apply(lambda x: TextBlob(x).sentiment.polarity)
        X_train_vectorized = np.concatenate((X_train_vectorized.toarray(), X_pol_train.to_numpy().reshape((-1, 1))), axis=1)
        X_test_vectorized = np.concatenate((X_test_vectorized.toarray(), X_pol_test.to_numpy().reshape((-1, 1))), axis=1)
          
        
    return X_train_vectorized, X_test_vectorized<jupyter_output><empty_output><jupyter_text>Model building and evaluation<jupyter_code># Build model and return f1 score and accuracy of specified model
  #input vectorized datasets and model tuning parameter, output f1 
def get_perform(classifier, X_train, y_train, X_test, y_test, MaxDepth=None, MaxFeatures='auto',C_para=1.0):
    if classifier == 'RandomForest':
        clf = RandomForestClassifier(random_state=RandomState,max_features=MaxFeatures,max_depth=MaxDepth)
    elif classifier == 'DecisionTree':
        clf = DecisionTreeClassifier(random_state=RandomState,max_features=MaxFeatures,max_depth=MaxDepth)
    elif classifier == 'SVC':
        clf = SVC(C=C_para,random_state=RandomState)
    elif classifier == 'Dummy':
        clf = DummyClassifier(strategy="uniform", random_state=RandomState)        
    else:
        print('Unexpected model type')
    clf.fit(X_train, y_train)
    preds = clf.predict(X_test)
    f1 = f1_score(y_test, preds, pos_label='positive',average='binary')

    return f1<jupyter_output><empty_output><jupyter_text>Compare performace with differenct models Raw data without preprocessing<jupyter_code>clf_models = ['RandomForest', 'DecisionTree','SVC', 'Dummy']
# Raw data with raw model --- no parameter tuned
X_train_raw, X_test_raw = get_vect(X_train, X_test, StopWords=None,MinDf=500,Ngram =(1,2), add_ft=False)
print('Performance of each model for un-processed data \n', )
for classifier in clf_models:
    f1_score_raw = get_perform(classifier, X_train_raw, y_train, X_test_raw, y_test,MaxDepth=None, MaxFeatures='auto',C_para=1.0)
    print(classifier,' classifier  \n f1 score: ', f1_score_raw, )<jupyter_output>Performance of each model for un-processed data 

RandomForest  classifier  
 f1 score:  0.7632232871849486
DecisionTree  classifier  
 f1 score:  0.6941534302129097
SVC  classifier  
 f1 score:  0.7734650670430486
Dummy  classifier  
 f1 score:  0.5396600566572237<jupyter_text>Apply lemmatization only<jupyter_code># Lemmatized text only

X_train_lem = X_train.apply(lambda x: preprocess_text(x))
X_test_lem = X_test.apply(lambda x: preprocess_text(x))

X_train_lem_vec, X_test_lem_vec = get_vect(X_train_lem, X_test_lem, StopWords=None,MinDf=500,Ngram =(1,2), add_ft=False)


print('Performance of each model for lemmatized only data \n', )
for classifier in clf_models:
    f1_score_lem_vec = get_perform(classifier, X_train_lem_vec, y_train, X_test_lem_vec, y_test, MaxDepth=None, MaxFeatures='auto',C_para=1.0)
    print(classifier,' classifier  \n f1 score: ', f1_score_lem_vec )<jupyter_output>Performance of each model for lemmatized only data 

RandomForest  classifier  
 f1 score:  0.766928011404134
DecisionTree  classifier  
 f1 score:  0.6983695652173914
SVC  classifier  
 f1 score:  0.7778165329612836
Dummy  classifier  
 f1 score:  0.5396600566572237<jupyter_text>Apply lemmatization and stopwords removal<jupyter_code># Lemmatized text with stopwords removal

X_train_lem_re = X_train.apply(lambda x: preprocess_text(x))#, lst_stopwords))
X_test_lem_re = X_test.apply(lambda x: preprocess_text(x))

X_train_lem_re_vec, X_test_lem_re_vec = get_vect(X_train_lem_re, X_test_lem_re, StopWords='english',MinDf=500,Ngram =(1,2), add_ft=False)
print('Performance of each model for lemmatized data and stopwords removal\n', )
for classifier in clf_models:
    f1_score_lem_re_vec = get_perform(classifier, X_train_lem_re_vec, y_train, X_test_lem_re_vec, y_test, MaxDepth=None, MaxFeatures='auto',C_para=1.0)
    print(classifier,' classifier  \n f1 score: ', f1_score_lem_re_vec )<jupyter_output>Performance of each model for lemmatized data and stopwords removal

RandomForest  classifier  
 f1 score:  0.7392455327597618
DecisionTree  classifier  
 f1 score:  0.7227912932138284
SVC  classifier  
 f1 score:  0.7368071689346165
Dummy  classifier  
 f1 score:  0.5396600566572237<jupyter_text>We can see that these is no significant improvement with lemmatization and stopwords removal.  Apply language detection<jupyter_code>raw_df['non_en'] = raw_df['review'].apply(lambda x: check_non_eng(x))

raw_df[raw_df.non_en == True].head(10)
en_df = raw_df[raw_df.non_en == False]

X_train_en, X_test_en, y_train_en, y_test_en = train_test_split(en_df.review, en_df.label, test_size=0.2, random_state=RandomState)

# Language detection

X_train_lem_en = X_train_en.apply(lambda x: preprocess_text(x))
X_test_lem_en = X_test_en.apply(lambda x: preprocess_text(x))


X_train_lem_en_vec, X_test_lem_en_vec = get_vect(X_train_lem_en, X_test_lem_en, StopWords=None,MinDf=500,Ngram =(1,2), add_ft=False)
print('Performance of each model for English-only data \n', )
for classifier in clf_models:
    f1_score_lem_en = get_perform(classifier, X_train_lem_en_vec, y_train_en, X_test_lem_en_vec, y_test_en, MaxDepth=None, MaxFeatures='auto',C_para=1.0)
    print(classifier,' classifier  \n f1 score: ', f1_score_lem_en )<jupyter_output>Performance of each model for English-only data 

RandomForest  classifier  
 f1 score:  0.7883315546069014
DecisionTree  classifier  
 f1 score:  0.705761316872428
SVC  classifier  
 f1 score:  0.8034965034965035
Dummy  classifier  
 f1 score:  0.5198711063372718<jupyter_text>There is slight improvement after we filter out reviews in other languages.  Add polarity as feature<jupyter_code>X_train_pol_vec, X_test_pol_vec = get_vect(X_train_lem_en, X_test_lem_en, StopWords=None,MinDf=500,Ngram =(1,2), add_ft=True)
print('Performance of each model after adding polarity as feature \n', )
for classifier in clf_models:
    f1_score_pol = get_perform(classifier, X_train_pol_vec, y_train_en, X_test_pol_vec, y_test_en, MaxDepth=None, MaxFeatures='auto',C_para=1.0)
    print(classifier,' classifier  \n f1 score: ', f1_score_pol )<jupyter_output>Performance of each model after adding polarity as feature 

RandomForest  classifier  
 f1 score:  0.8063241106719368
DecisionTree  classifier  
 f1 score:  0.7446736557321609
SVC  classifier  
 f1 score:  0.8236963462220646
Dummy  classifier  
 f1 score:  0.5198711063372718
{'fraction_non_alphanumeric': 0.09472246610086821, 'fraction_numerical': 0.04341039898546483, 'mean_word_length': 5.383561643835616, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '378725', 'license': 'MIT', 'n_tokens_mistral': 3925, 'n_tokens_neox': 3657, 'n_words': 833, 'path': 'playground/bojia/Sentiment analysis --DT,RF,SVC.ipynb', 'repo_name': 'mads-app-reviews-nlp/app-reviews-nlp', 'size': '10251'}