<jupyter_start><jupyter_text>Sentence classification task<jupyter_code># RUN ONCE load models + data
#!python -m spacy download en_core_web_md
#import nltk
#nltk.download('stopwords')

# get data via wget if it doesn't exist
#!wget -nc http://öä.eu/sst5.data.txt
#!wget -nc http://öä.eu/sst5.labels.txt

# other option: load data via gdrive mounting
# drive.mount('/gdrive')
# %cd /gdrive/My Drive/sst5/<
%matplotlib inline
import numpy as np
import pandas as pd
import seaborn as sns
import numpy as np
import spacy 
import sklearn
import nltk
from nltk.corpus import stopwords
from nltk.stem.porter import *
from collections import Counter
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.base import TransformerMixin 
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
from sklearn.model_selection import train_test_split
from sklearn.decomposition import TruncatedSVD, PCA
from spacy.lang.en.stop_words import STOP_WORDS
from spacy.lang.en import English
from spacy.lemmatizer import Lemmatizer
from spacy.tokens import Doc, Span, Token
from matplotlib import pyplot as plt
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV
flatten = lambda l: [item for sublist in l for item in sublist]
mapl =    lambda f, i: list(map(f, i))
filterl = lambda f, i: list(filter(f, i))
SEED = 42<jupyter_output><empty_output><jupyter_text>**Data Exploration** ***sst-5 is a sentiment analysis collection, created based on the movie reviews in the IMDBwebsite [1]. The provided dataset in this assignment consists of a set of sentences, and theircorresponding labels. The labels are in 5 levels, annotated from very negative (-2) to verypositive (+2). The dataset is available for downloading in http://drive.jku.at/ssf/s/readFile/share/14324/4544703232741430166/publicLink/sst5.zip***<jupyter_code>df = pd.read_csv('sst5.data.txt')
X = np.array(df['text'])
Y = np.array(df['label'])
df<jupyter_output><empty_output><jupyter_text>The data also has the issue, that the classes -1 and 1 have significantly more members: Over 6000 in these 2 vs 500 in the other 3 classes.<jupyter_code>df['label'].hist()
labels = pd.read_csv('sst5.labels.txt',header=None)
label_id = {lab[0]: lab[1] for lab in labels.to_numpy()}; label_id<jupyter_output><empty_output><jupyter_text>**Manual data inspection for adjectives**<jupyter_code># loading a prebuilt the english language processing pipeline - that starts out with 3 components. this can take some time
nlp = spacy.load("en_core_web_md")
print(*nlp.pipeline, sep = "\n")
# Loop to get just the adjectives from each review
docl = list( nlp.pipe(X)) # returns just an iterator -> make list
adj_filter = lambda tok: tok.pos_ == 'ADJ'
words_adj = [filterl(adj_filter, line) for line in docl]
df['words_adj'] = pd.Series(words_adj)
df.head() # we can see that for sentence 1 that adj were found which could be because of the default dictionary not having all words 
max_sent_len=max(len(docl[i]) for i in range(0,len(docl)))
print('maximum sentence length = ', max_sent_len)<jupyter_output>maximum sentence length =  56<jupyter_text>Analyze the most common adjectives and plot histograms<jupyter_code># Print the most common adjectives
adj_count = dict()
for id in label_id:
    adj = df[df['label']==id]['words_adj']
    print(adj)
    adj = flatten(adj)
    adj = mapl(lambda tok: tok.lemma_, adj)
    adj_count[id] = Counter(adj).most_common(10)
    print(label_id[id],adj_count[id])

#Potting of all review types
def plot_reviews(adj_count,reviewtype):
    labels, values = zip(*adj_count)
    indexes = np.arange(len(labels))

    #sns.set(style="whitegrid")
    ax = sns.barplot(indexes, values, palette="BuGn_r")

    plt.xticks(indexes + 0.1, labels, rotation=30);
    plt.tick_params(axis='x', labelsize=13)
    plt.title("Most Common "+str(reviewtype)+" Adjectives in Reviews")
    plt.show()


for id in label_id:
    plot_reviews(adj_count[id],label_id[id])<jupyter_output>4               [honest, entire, crude, exploitative]
16                                             [lead]
36                      [own, sadistic, sour, little]
37                                                 []
41       [loose, unaccountable, sophisticated, worst]
                             ...                     
11797                            [elemental, genuine]
11800                                [slapstick, big]
11806                          [first, second, small]
11821                        [small, natural, pretty]
11842                                   [meaningless]
Name: words_adj, Length: 1508, dtype: object
very negative [('bad', 147), ('good', 46), ('dull', 41), ('more', 38), ('much', 29), ('own', 28), ('old', 28), ('little', 27), ('funny', 27), ('many', 25)]
2                        [dry, philosophical, sedate]
13               [cold, intricate, difficult, deeper]
17                                                 []
22                                         [...]<jupyter_text>Task1: Dataset Preparation If you want class-balancing for task 4.3, rerun this notebook with this cell enabled minl = sum(df["label"]==-2)subsamples = [df[df["label"]==i][:minl] for i in range(-2, 3)]df = pd.concat(subsamples)df['label'].hist()X = np.array(df['text'])Y = np.array(df['label'])docl = list( nlp.pipe(X))  create new smaller docl  Preprocessing. Most of this is done by the en_core_web_sm piepline, that contains an english tokenizer, a tagger that cleans up unneccesary spaces, a parser and a named entity recognizer and has built in stopword detection.But we can improve it by adding NLTK stemming and building a custom dict. Stopwords - can be checked with tok.is_stop()<jupyter_code># use spacy and nltk stopwords and put them into the default nlp stopword detector
print("Adding nltk stopwords", set(nltk.corpus.stopwords.words('english')).difference(nlp.Defaults.stop_words))
nlp.Defaults.stop_words = nlp.Defaults.stop_words.union(nltk.corpus.stopwords.words('english')); 
print("Total list of stopwords", ", ".join(sorted(nlp.Defaults.stop_words)))<jupyter_output>Adding nltk stopwords {'having', 'aren', 'isn', 'wouldn', 'mustn', "mustn't", 'didn', 've', 'mightn', 'y', 'hadn', "she's", "hadn't", 'needn', 'don', "don't", "won't", 'wasn', "wouldn't", "mightn't", "haven't", "wasn't", "didn't", 'couldn', 'shouldn', "needn't", 'hasn', 'ma', "weren't", "couldn't", 'doesn', 'ain', 't', 'won', 'd', "you're", 'm', "you'd", "that'll", "doesn't", 's', "shouldn't", "aren't", 'weren', "isn't", "it's", "should've", 'shan', "shan't", "hasn't", 'theirs', "you've", 'll', "you'll", 'haven', 'o'}
Total list of stopwords 'd, 'll, 'm, 're, 's, 've, a, about, above, across, after, afterwards, again, against, ain, all, almost, alone, along, already, also, although, always, am, among, amongst, amount, an, and, another, any, anyhow, anyone, anything, anyway, anywhere, are, aren, aren't, around, as, at, back, be, became, because, become, becomes, becoming, been, before, beforehand, behind, being, below, beside, besides, between, beyond, both, bottom, but, by, ca, call, c[...]<jupyter_text>To add a custom stemmer, you can extend tokens with getters<jupyter_code>stem_function = PorterStemmer().stem
Token.set_extension("stem", getter=lambda tok: stem_function(tok.lemma_), force=True) # force=True to be able to rerun the line<jupyter_output><empty_output><jupyter_text>Print 3 sentences with to show lemmatization, stemming, stopwords and more:<jupyter_code>print("Word\tLemma\tStem\tstop")
for d in docl[:3]:
    for word in d:
        print(word, word.lemma_, word._.stem, word.is_stop or word.is_punct, sep="\t")
    print() # newline between sentences<jupyter_output>Word	Lemma	Stem	stop
Reno	Reno	reno	False
himself	-PRON-	-pron-	True
can	can	can	True
take	take	take	True
credit	credit	credit	False
for	for	for	True
most	most	most	True
of	of	of	True
the	the	the	True
movie	movie	movi	False
's	's	's	True
success	success	success	False
.	.	.	True

Despite	despite	despit	False
the	the	the	True
film	film	film	False
's	's	's	True
shortcomings	shortcoming	shortcom	False
,	,	,	True
the	the	the	True
stories	story	stori	False
are	be	be	True
quietly	quietly	quietli	False
moving	move	move	False
.	.	.	True

Despite	despite	despit	False
its	-PRON-	-pron-	True
dry	dry	dri	False
wit	wit	wit	False
and	and	and	True
compassion	compassion	compass	False
,	,	,	True
the	the	the	True
film	film	film	False
suffers	suffer	suffer	False
from	from	from	True
a	a	a	True
philosophical	philosophical	philosoph	False
emptiness	emptiness	empti	False
and	and	and	True
maddeningly	maddeningly	maddeningli	False
sedate	sedate	sedat	False
pacing	pacing	pace	False
.	.	.	True<jupyter_text>Creating a dictionary.To reduce the dict size we use lemmatization and stopword/punctuation removal. This leads to 250k -> 15k -> 12k vocabulary size.<jupyter_code>vocab0 = set(flatten(docl))
print("Size without any filtering/stemming", len(vocab0))
print("Examples", list(vocab0)[:50])

vocab1 = set(flatten([map(lambda tok: tok.lemma_, line) for line in docl]))
print("Size with lemmatization alone", len(vocab1))
print("Examples", list(vocab1)[:50])

stemf = lambda tok: tok._.stem
tokfilt = lambda tok: not tok.is_stop and not tok.is_punct

stem_doc = [ mapl(stemf, # only use stem
                  filter(tokfilt, line)) # filter out punctuation
             for line in docl]
vocab2 = set(flatten(stem_doc)) # make unique set out of list of list
print("Size with filtering/stemming", len(vocab2))
print("Examples", list(vocab2)[:50])<jupyter_output>Size without any filtering/stemming 235066
Examples [running, brutally, `, 're, ,, too, it, ideas, Francophiles, This, -, Nature, like, there, quite, Nothing, ,, like, big, Gloriously, incurably, or, ,, Fire, of, several, its, best, Weaver, in, worst, conditioning, and, A, intimate, pounds, is, dialogue, should, and, observe, of, are, the, it, love, n't, this, the, around]
Size with lemmatization alone 15800
Examples ['spectrum', 'dogged', 'Original', 'discouraging', 'scenario', 'pedestal', 'Collosum', 'litmus', 'simultaneously', 'Tolstoy', 'blisteringly', 'substantive', 'sampling', 'coast', 'just', 'efficient', 'wither', 'have', 'unease', 'dive', 'humourless', 'exhibit', 'clamor', 'labour', 'sure', 'southern', 'Dolby', 'focused', 'trumpet', 'nrelentingly', 'Cox', 'age', 'ferocious', 'pasta', 'delicately', 'contemptuous', 'Neil', 'backmasking', 'exit', 'Le', 'Orleans', 'monster\\/science', 'extraordinarily', 'forth', 'sized', 'conspicuously', 'relocate', 'Saldanha', 'courage', 'meld']
[...]<jupyter_text>Splitting the dataset.<jupyter_code>text_train, text_test, label_train, label_test = train_test_split(docl,Y,test_size = 0.2,random_state = SEED)
print(*zip(label_train[:10], text_train[:10]), sep="\n")<jupyter_output>(-1, Care deftly captures the wonder and menace of growing up , but he never really embraces the joy of Fuhrman 's destructive escapism or the grace-in-rebellion found by his characters .)
(0, ... Ice Age treads predictably along familiar territory , making it a passable family film that wo n't win many fans over the age of 12 .)
(-1, Oft-described as the antidote to American Pie-type sex comedies , it actually has a bundle in common with them , as the film diffuses every opportunity for a breakthrough)
(-2, Some writer dude , I think his name was , uh , <NAME> , was supposed to have like written the screenplay or something , but , dude , the only thing that I ever saw that was written down were the zeroes on my paycheck .)
(1, It has plenty of laughs .)
(1, Grant carries the day with impeccable comic timing , raffish charm and piercing intellect .)
(-1, When the fire burns out , we 've only come face-to-face with a couple dragons and that 's where the film ultimately fails .)
[...]<jupyter_text>Task2: Feature Extraction  Creating sentence vectors. We are using [TF-IDF (Term-Frequency, Inverse-Document-Frequency)](https://en.wikipedia.org/wiki/Tf%E2%80%93idf) appraoch which includes the term and document frequencies for term weighting. Our 2nd approach is the spacy builtin [word2vec](https://en.wikipedia.org/wiki/Word2vec) which only has implicit pretrained weighting.  **word-ids**<jupyter_code>sec2vec = {word:i for i,word in enumerate(vocab0)}<jupyter_output><empty_output><jupyter_text>TF-idf with scikit learn<jupyter_code># as our data is already tokenized, we can skip tokenization by setting the tokenizer to the identity function and skipping lowercase
tfvectorizer = TfidfVectorizer(vocabulary=vocab2, tokenizer=lambda x: x, lowercase=False);
tfdoc = tfvectorizer.fit_transform(stem_doc)
print(tfvectorizer); tfdoc
print("The sparcity rate (rate of emtpy elements) is", 100*(1 - len(tfdoc.nonzero()[0])/np.prod(tfdoc.shape)), "%")<jupyter_output>The sparcity rate (rate of emtpy elements) is 99.9291354216198 %<jupyter_text>As this array is 11844x12081, you can't use PCA which is why we can only use TruncatedSVD  Word2Vec with spacySpacy also offers a built in word2vec in all models bigger than small, and word2vec is the state of the art for text vectorization. The vec size is 96<jupyter_code>w2v = np.array(mapl(lambda tok: tok.vector, docl)); w2v.shape<jupyter_output><empty_output><jupyter_text>Dimensionality reduction.<jupyter_code>tf_20 = TruncatedSVD(n_components=20).fit_transform(tfdoc)
w2v_20 = PCA(n_components=20).fit_transform(w2v)<jupyter_output><empty_output><jupyter_text>Task3: Classification  Training classifiers.<jupyter_code>def train(clf,x_train,y_train,hyperparameter):
    """
    clf: The specified model
    x_train:The sentence vector from the training dataset
    y_train:The corresponding labels from the training dataset
    hyperparameter:Dictionary with parameters names (string) as keys and lists of parameter settings to try as values.
    """
    classifier = GridSearchCV(clf, hyperparameter,n_jobs=8,cv=10)
    grid_result = classifier.fit(x_train,y_train)
    print('Score: ',grid_result.best_score_)
    return grid_result.best_estimator_
print('Baseline = ',max(list(Counter(Y).values()))/sum(list(Counter(Y).values())))<jupyter_output>Baseline =  0.2648598446470787<jupyter_text>**Classifers**<jupyter_code>names = ['Gradient boosting','Random Forest','Linear Support vector','SGD Classifier']
model_list = [GradientBoostingClassifier(),RandomForestClassifier(),LinearSVC(),SGDClassifier()]
hyperparameters = [{'learning_rate':[0.1,0.01,0.001]}, 
                   {'n_estimators':[100,1000],'random_state':[42],'class_weight':("balanced",None)},
                   {'loss':('hinge','squared_hinge'),'C':[1.0,8.0,10.0],'random_state':[42]},
                   {'loss':('hinge','squared_epsilon_insensitive'),'epsilon':[0.1,0.25,0.01]}]<jupyter_output><empty_output><jupyter_text>Results and comparison. Report the evaluation results of the models on both validation and test sets in a table. Compare different models, and conclude the best performing approach on the dataset.<jupyter_code>def scores(clf, X_val, y_val,X_test,y_test):
    """
    Input:
        clf: The model
        X_val:The sentence vector from the validation dataset
        y_val: The corresponding labels from the validation dataset
        X_test:The sentence vector from the test dataset
        y_test:The corresponding labels from the test dataset
    Output:
        acc_val: Accuracy for the validation set
        acc_test: Accuracy for the test set 
        conf_mat: Confusion matrix for the validation set
        conf_mat_test: confusion matrix for the test set 
    """
    confidence = clf.score(X_val,y_val)
    y_pred = clf.predict(X_val)
    acc_val = accuracy_score(y_val, y_pred)
    conf_mat = confusion_matrix(y_val, y_pred)
    test_labels_pred = clf.predict(X_test)
    acc_test = accuracy_score(y_test, test_labels_pred)
    conf_mat_test = confusion_matrix(y_test, test_labels_pred)
    return acc_val,acc_test,conf_mat,conf_mat_test

def evaluate(model_list,vectors,hyperparameters,names):
    """
    Input:
        model_list: A list which contains all initilized models
        vectors: A dictionary which contains the name of the sentence vector as keys and sentence vector variable as a value
        hyperparameters: A list of dictionaries that contain the parameters for each model 
        names: A list containg the names of the models as strings
    Output:
        table: pandas dataframe which displays the classifers the corrresp[onding sentence vectors and how they performed
        conf_list: A list which contains the corresponding confusion matrix for each classifier
    """
    test_acc_list = []
    val_acc_list=[]
    sentence_vector_list = []
    conf_list = []
    name_list = []
    for key in vectors.keys():
        i=0
        print('Sentence Vector',key)
        x_train,x_test,y_train,y_test = train_test_split(vectors[key],Y,test_size=.2,random_state=42)
        x_train,x_val,y_train,y_val = train_test_split(x_train,y_train,test_size=.2,random_state=42)
        for model in model_list:
            print('classifier = ',names[i])
            clf = train(model,x_train,y_train,hyperparameters[i])
            #print('finished training')
            val_acc,test_acc,conf_val,conf_test= scores(clf,x_val,y_val,x_test,y_test)
            test_acc_list.append(test_acc)
            val_acc_list.append(val_acc)
            sentence_vector_list.append(key)
            name_list.append(names[i])
            conf_list.append((conf_val,conf_test))
            i+=1
    table = pd.DataFrame({'Model':model_list+model_list,
                    'sentence vector':sentence_vector_list,
                    'Validation Accuracy ':val_acc_list,
                    'Test Accuracy':test_acc_list})

    return table, conf_list

vectors = {'tf-id':tf_20,'word2vec':w2v_20}
result, conf_list= evaluate(model_list,vectors,hyperparameters,names)
result
def plot_confusion_matrix(num_classifer,conf_list):
    """
    Input:
        num_classifer: The classifer which we want to see the confusion matrix of
        conf_list: A list of confusion matrices corresponding to each classifier in the table above
    Output:
        Two confusion matrix plots of both the validation dataset and the test dataset 
    """
    conf_mat, conf_mat_test = conf_list[num_classifer]
    for name, data in [("Validation Data", conf_mat), ("Test data", conf_mat_test)]:
        # Plot Confusion Matrix Data as a Matrix
        plt.matshow(data)
        plt.title('Confusion Matrix for '+name)
        plt.colorbar()
        plt.ylabel('True label')
        plt.xlabel('Predicted label')
        plt.show()
print(len(conf_list))
plot_confusion_matrix(6,conf_list) # Plots the confusion matrix of the best model based on the test accuracy score shown above<jupyter_output>8<jupyter_text>Summary Findings: Explanation of behaviourLooking at the matrix above and the results table in the Results and comparison section, we can see that the accuracy scores that we got when evaluating the model are due to our model focusing on two classes mainly out of the 5 classes in this dataset. We can see that our model predicts the best on class 1(negative) and class 3 (positive) best. This performance can be due to many factors that include the data distribution and the vocabulary in the specific classes. As shown during our Data exploration section of our notebook there are many overlapping words between the classes which further contributed to the accuracy score that we got for this task. Furthermore we can also see that the model somewhat performs well on class 4 also (very positive) which again could be due to the overlap in features between the classes.  The model doesnt perform so well in predicting coirrectly for the class 0 and 2 which are very negative and neutral respectively, which could be due to the vocab in these classes doesn't occur enough in the samples or they include characteristucs like sarcasm which is not easy to detect.  **Task4: Visualization and Analysis**   QuestionsWe want to find out two things about our preprocessing:1. Does the stemming + lemmatization preprocessing (used with TF-IDF) actually help?2. Does applying PCA to to our word2vec help?3. Would balacing the classes help?<jupyter_code>#tF_idf + random forest, trying some other preprocessing like the none-preprocessing
tfvectorizer = TfidfVectorizer(vocabulary=vocab0, tokenizer=lambda x: x, lowercase=False)
tfdoc = tfvectorizer.fit_transform(stem_doc)
tf_20 = TruncatedSVD(n_components=20).fit_transform(tfdoc)

# word2vec, if it performs differently if we don't PCA it in between
names = ['Gradient boosting','Random Forest','Linear Support vector','SGD Classifier']
model_list = [GradientBoostingClassifier(),RandomForestClassifier(),LinearSVC(),SGDClassifier()]
hyperparameters = [{'learning_rate':[0.1,0.01,0.001]},
                   {'n_estimators':[100,1000],'random_state':[42],'class_weight':("balanced",None)},
                   {'loss':('hinge','squared_hinge'),'C':[1.0,8.0,10.0],'random_state':[42]},
                   {'loss':('hinge','squared_epsilon_insensitive'),'epsilon':[0.1,0.25,0.01]}]

vectors = {'tf-id':tf_20,'word2vec':w2v}
result,conf_list= evaluate(model_list,vectors,hyperparameters,names); result
plot_confusion_matrix(1,conf_list) #confusion matrix for Random Forest + TF-ID
plot_confusion_matrix(5,conf_list) #confusion matrix for best model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1092815065633842, 'fraction_numerical': 0.017347743401827542, 'mean_word_length': 4.460992907801418, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 68, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '294463', 'license': 'MIT', 'n_tokens_mistral': 7224, 'n_tokens_neox': 6779, 'n_words': 2184, 'path': 'Sentence_classification_task.ipynb', 'repo_name': 'fathyshalaby/Somemoredeeplearning', 'size': '21567'}