<jupyter_start><jupyter_text>Loading the data<jupyter_code>final_data = pd.read_csv("final.csv")

final_data = final_data.drop(["Text"], axis = 1)
final_data = final_data.drop(final_data.columns[0], axis = 1)<jupyter_output><empty_output><jupyter_text>Sorting and train/test split<jupyter_code>labels = final_data.Score
final_data = final_data.sort_values("Time")
final_data.shape
n = final_data.shape[0]
train_size = 0.7

train_set = final_data.iloc[:int(n*train_size)]
test_set =final_data.iloc[int(n*train_size):]

X_train = train_set.CleanedText
y_train = train_set.Score

X_test = test_set.CleanedText
y_test= test_set.Score<jupyter_output><empty_output><jupyter_text>Cleaning the data<jupyter_code>import nltk
nltk.download('stopwords')
import re
# Tutorial about Python regular expressions: https://pymotw.com/2/re/
import string
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.stem.wordnet import WordNetLemmatizer

stop = set(stopwords.words('english')) #set of stopwords
sno = nltk.stem.SnowballStemmer('english') #initialising the snowball stemmer

def cleanhtml(sentence): #function to clean the word of any html-tags
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, ' ', sentence)
    return cleantext
def cleanpunc(sentence): #function to clean the word of any punctuation or special characters
    cleaned = re.sub(r'[?|!|\'|"|#]',r'',sentence)
    cleaned = re.sub(r'[.|,|)|(|\|/]',r' ',cleaned)
    return  cleaned
print(stop)
print('************************************')
print(sno.stem('tasty'))<jupyter_output>[nltk_data] Downloading package stopwords to /content/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
{'am', 'below', 'through', 'mustn', 'mightn', 'itself', "didn't", 'm', 'the', 'haven', 'couldn', 'no', 's', 'most', 'me', 'herself', 'did', 'during', "couldn't", 'for', 'to', "won't", 'until', 'have', 'its', "doesn't", 'ourselves', "she's", 'theirs', 'aren', 'hadn', 'isn', 'but', 'just', 'at', 'hers', 're', 'weren', 'ours', "you're", 'does', 'd', 'his', 'ma', 'myself', 'on', 'has', 'we', 'now', 'a', 'himself', 'after', 'shan', 'once', 'who', 'or', "shan't", 'some', 'this', 'off', 'yourself', 'do', 'before', 't', 'as', 'of', 'because', 'will', 'y', 'needn', 've', 'her', 'they', "shouldn't", 'with', 'him', 'other', "it's", "isn't", 'yourselves', 'over', 'them', 'under', 'when', 'too', 'is', 'had', 'above', "haven't", 'same', 'our', 'about', 'being', 'didn', 'wouldn', "aren't", 'those', 'are', 'while', 'o', "don't", 'only', 'what', 'so', 'more', 'own', 'very', 'their',[...]<jupyter_text>- - -  Creating Word2vec model<jupyter_code>import gensim
i=0
train_sent=[]
for sent in X_train:
    filtered_sentence=[]
    sent=cleanhtml(sent)
    for w in sent.split():
        for cleaned_words in cleanpunc(w).split():
            if(cleaned_words.isalpha()):    
                filtered_sentence.append(cleaned_words.lower())
            else:
                continue 
    train_sent.append(filtered_sentence)
    
test_sent=[]
for sent in X_test:
    filtered_sentence=[]
    sent=cleanhtml(sent)
    for w in sent.split():
        for cleaned_words in cleanpunc(w).split():
            if(cleaned_words.isalpha()):    
                filtered_sentence.append(cleaned_words.lower())
            else:
                continue 
    test_sent.append(filtered_sentence)
from gensim.models import Word2Vec
from gensim.models import KeyedVectors

w2v_model = gensim.models.Word2Vec(train_sent,min_count=5,size=50, workers=4)<jupyter_output><empty_output><jupyter_text>- - -  Average Word2Vec<jupyter_code>#AVG-W2V
sent_vectors = []; # the avg-w2v for each sentence/review is stored in this list
for sent in train_sent: # for each review/sentence
    sent_vec = np.zeros(50) # as word vectors are of zero length
    cnt_words =0; # num of words with a valid vector in the sentence/review
    for word in sent: # for each word in a review/sentence
        try:
            vec = w2v_model.wv[word]
            sent_vec += vec
            cnt_words += 1
        except:
            cnt_words = 1
            pass
    sent_vec /= cnt_words
    sent_vectors.append(sent_vec)
print(len(sent_vectors))
print(len(sent_vectors[0]))
sent_vectors2 = []; # the avg-w2v for each sentence/review is stored in this list
for sent in test_sent: # for each review/sentence
    sent_vec = np.zeros(50) # as word vectors are of zero length
    cnt_words =0; # num of words with a valid vector in the sentence/review
    for word in sent: # for each word in a review/sentence
        try:
            vec = w2v_model.wv[word]
            sent_vec += vec
            cnt_words += 1
        except:
            cnt_words = 1
            pass
    sent_vec /= cnt_words
    sent_vectors2.append(sent_vec)
print(len(sent_vectors2))
print(len(sent_vectors2[0]))
X_train1 = sent_vectors
X_test1 = sent_vectors2<jupyter_output><empty_output><jupyter_text>Applying DecisionTree Classifier<jupyter_code>from sklearn.tree import DecisionTreeClassifier


depth = list(range(2,10))


# empty list that will hold cv scores
cv_scores = []
my_cv = [(train,test) for train, test in TimeSeriesSplit(n_splits=10).split(X_train1)]
# perform 10-fold cross validation
for d in depth:
    dt = DecisionTreeClassifier(max_depth = d, min_samples_split = 1000, min_samples_leaf = 500)
    scores = cross_val_score(dt, X_train1, y_train, cv = my_cv, scoring='accuracy')
    cv_scores.append(scores.mean())
    
    
# changing to misclassification error
MSE = [1 - x for x in cv_scores]

# determining best k
optimal_d = depth[MSE.index(min(MSE))]
print('\nThe optimal depth of the tree is %d.' % optimal_d)

# plot misclassification error vs k 
plt.plot(depth, MSE)

for xy in zip(depth, np.round(MSE,3)):
    plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')

plt.xlabel('Tree depth')
plt.ylabel('Misclassification Error')
plt.show()


dt = DecisionTreeClassifier(max_depth = 8, min_samples_split = 1000, min_samples_leaf = 500)
dt.fit(X_train1,y_train)
pred = dt.predict(X_test1)

acc = accuracy_score(y_test, pred, normalize=True) * float(100)
x = dt.predict(X_train1)
tr_acc = accuracy_score(y_train, x, normalize=True) * float(100)
print('\n****Train accuracy for k = {} is {:.2f}'.format(8,tr_acc))
print('\n****Test accuracy for k = {} is {:.2f}'.format(8,acc))<jupyter_output>****Train accuracy for k = 8 is 87.32

****Test accuracy for k = 8 is 85.31<jupyter_text>- - -   Word2Vec-Tfidf Note : Performed with sampled 50k datapoints<jupyter_code>from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import TfidfVectorizer


tf_idf_vect = TfidfVectorizer()
final_tf_idf = tf_idf_vect.fit_transform(X_train)
tfidf_feat = tf_idf_vect.get_feature_names()
 # tfidf words/col-names
# final_tf_idf is the sparse matrix with row= sentence, col=word and cell_val = tfidf

train_vectors = []; # the tfidf-w2v for each sentence/review is stored in this list
row=0;
for sent in train_sent: # for each review/sentence
    sent_vec = np.zeros(50) # as word vectors are of zero length
    weight_sum = 0; # num of words with a valid vector in the sentence/review
    for word in sent: # for each word in a review/sentence
        try:
            vec = w2v_model.wv[word]
            # obtain the tf_idf of a word in a sentence/review
            tfidf = final_tf_idf[row, tfidf_feat.index(word)]
            sent_vec += (vec * tfidf)
            weight_sum += tfidf
            
            
        except:
            weight_sum = 1
            pass

    sent_vec /= weight_sum
    #print(np.isnan(np.sum(sent_vec)))

    train_vectors.append(sent_vec)
    row += 1
print(len(train_vectors))
print(len(train_vectors[0]))

final_tf_idf = tf_idf_vect.fit_transform(X_test)
tfidf_feat = tf_idf_vect.get_feature_names()
 # tfidf words/col-names
# final_tf_idf is the sparse matrix with row= sentence, col=word and cell_val = tfidf

test_vectors = []; # the tfidf-w2v for each sentence/review is stored in this list
row=0;
for sent in test_sent: # for each review/sentence
    sent_vec = np.zeros(50) # as word vectors are of zero length
    weight_sum = 0; # num of words with a valid vector in the sentence/review
    for word in sent: # for each word in a review/sentence
        try:
            vec = w2v_model2.wv[word]
            # obtain the tf_idf of a word in a sentence/review
            tfidf = final_tf_idf[row, tfidf_feat.index(word)]
            sent_vec += (vec * tfidf)
            weight_sum += tfidf
            
            
        except:
            weight_sum = 1
            pass

    sent_vec /= weight_sum
    #print(np.isnan(np.sum(sent_vec)))

    test_vectors.append(sent_vec)
    row += 1
print(len(test_vectors))
print(len(test_vectors[0]))
X_train2 = train_vectors
X_test2 = test_vectors<jupyter_output><empty_output><jupyter_text>Applying DecisionTree Classifier<jupyter_code>from sklearn.tree import DecisionTreeClassifier


depth = list(range(2,15))

# empty list that will hold cv scores
cv_scores = []
my_cv = [(train,test) for train, test in TimeSeriesSplit(n_splits=10).split(X_train2)]
# perform 10-fold cross validation
for d in depth:
    dt = DecisionTreeClassifier(max_depth = d, min_samples_split = 500, min_samples_leaf = 100)
    scores = cross_val_score(dt, X_train2, y_train, cv = my_cv, scoring='accuracy')
    cv_scores.append(scores.mean())
    
    
# changing to misclassification error
MSE = [1 - x for x in cv_scores]

# determining best k
optimal_d = depth[MSE.index(min(MSE))]
print('\nThe optimal depth of the tree is %d.' % optimal_d)

# plot misclassification error vs k 
plt.plot(depth, MSE)

for xy in zip(depth, np.round(MSE,3)):
    plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')

plt.xlabel('Tree depth')
plt.ylabel('Misclassification Error')
plt.show()

dt = DecisionTreeClassifier(max_depth = 6, min_samples_split = 500, min_samples_leaf = 100)
dt.fit(X_train2,y_train)
pred = dt.predict(X_test2)

acc = accuracy_score(y_test, pred, normalize=True) * float(100)
x = dt.predict(X_train2)
tr_acc = accuracy_score(y_train, x, normalize=True) * float(100)
print('\n****Train accuracy for k = {} is {:.2f}'.format(6,tr_acc))
print('\n****Test accuracy for k = {} is {:.2f}'.format(6,acc))<jupyter_output>****Train accuracy for k = 6 is 86.09

****Test accuracy for k = 6 is 83.13
{'fraction_non_alphanumeric': 0.13873055743406434, 'fraction_numerical': 0.015554052748526712, 'mean_word_length': 3.960229995208433, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '21803', 'license': 'MIT', 'n_tokens_mistral': 3855, 'n_tokens_neox': 3635, 'n_words': 921, 'path': 'Decision Trees on amazon reviews.ipynb', 'repo_name': 'kunwar-vikrant/Decision-Trees-on-Amazon-Reviews-dataset', 'size': '10351'}