ositive = True for line in content.split("

"): if "positive" in line.lower(): positive = True continue elif "negative" in line.lower(): positive = False continue self.emoticons[line] = positive def is_positive(self, emoticon): if emoticon in self.emoticons: return self.emoticons[emoticon] return False def is_emoticon(self, to_check): return to_check in self.emoticons

class TwitterData_ExtraFeatures(TwitterData_Wordlist): def __init__(self): pass def build_data_model(self): extra_columns = [col for col in self.processed_data.columns if col.startswith("number_of")] label_column = [] if not self.is_testing: label_column = ["label"] columns = label_column + extra_columns + list( map(lambda w: w + "_bow",self.wordlist)) labels = [] rows = [] for idx in self.processed_data.index: current_row = [] if not self.is_testing: # add label current_label = self.processed_data.loc[idx, "emotion"] labels.append(current_label) current_row.append(current_label) for _, col in enumerate(extra_columns): current_row.append(self.processed_data.loc[idx, col]) # add bag-of-words tokens = set(self.processed_data.loc[idx, "text"]) for _, word in enumerate(self.wordlist): current_row.append(1 if word in tokens else 0) rows.append(current_row) self.data_model = pd.DataFrame(rows, columns=columns) self.data_labels = pd.Series(labels) return self.data_model, self.data_labels def build_features(self): def count_by_lambda(expression, word_array): return len(list(filter(expression, word_array))) def count_occurences(character, word_array): counter = 0 for j, word in enumerate(word_array): for char in word: if char == character: counter += 1 return counter def count_by_regex(regex, plain_text): return len(regex.findall(plain_text)) self.add_column("splitted_text", map(lambda txt: txt.split(" "), self.processed_data["text"])) # number of uppercase words uppercase = list(map(lambda txt: count_by_lambda(lambda word: word == word.upper(), txt), self.processed_data["splitted_text"])) self.add_column("number_of_uppercase", uppercase) # number of ! exclamations = list(map(lambda txt: count_occurences("!", txt), self.processed_data["splitted_text"])) self.add_column("number_of_exclamation", exclamations) # number of ? questions = list(map(lambda txt: count_occurences("?", txt), self.processed_data["splitted_text"])) self.add_column("number_of_question", questions) # number of ... ellipsis = list(map(lambda txt: count_by_regex(regex.compile(r"\.\s?\.\s?\."), txt), self.processed_data["text"])) self.add_column("number_of_ellipsis", ellipsis) # number of hashtags hashtags = list(map(lambda txt: count_occurences("#", txt), self.processed_data["splitted_text"])) self.add_column("number_of_hashtags", hashtags) # number of mentions mentions = list(map(lambda txt: count_occurences("@", txt), self.processed_data["splitted_text"])) self.add_column("number_of_mentions", mentions) # number of quotes quotes = list(map(lambda plain_text: int(count_occurences("'", [plain_text.strip("'").strip('"')]) / 2 + count_occurences('"', [plain_text.strip("'").strip('"')]) / 2), self.processed_data["text"])) self.add_column("number_of_quotes", quotes) # number of urls urls = list(map(lambda txt: count_by_regex(regex.compile(r"http.?://[^\s]+[\s]?"), txt), self.processed_data["text"])) self.add_column("number_of_urls", urls) # number of positive emoticons ed = EmoticonDetector() positive_emo = list( map(lambda txt: count_by_lambda(lambda word: ed.is_emoticon(word) and ed.is_positive(word), txt), self.processed_data["splitted_text"])) self.add_column("number_of_positive_emo", positive_emo) # number of negative emoticons negative_emo = list(map( lambda txt: count_by_lambda(lambda word: ed.is_emoticon(word) and not ed.is_positive(word), txt), self.processed_data["splitted_text"])) self.add_column("number_of_negative_emo", negative_emo) def add_column(self, column_name, column_content): self.processed_data.loc[:, column_name] = pd.Series(column_content, index=self.processed_data.index)

data = TwitterData_ExtraFeatures() data.initialize("data\\train.csv") data.build_features() data.cleanup(TwitterCleanuper()) data.tokenize() data.stem() data.build_wordlist() data_model, labels = data.build_data_model()

Logic behind extra features

Let's see how (some) of the extra features separate the data set. Some of them, i.e number exclamation marks, number of pos/neg emoticons do this really well. Despite of the good separation, those features sometimes occur only on small subset of the training dataset.

sentiments = ["positive","negative","neutral"] plots_data_ef = [] for what in map(lambda o: "number_of_"+o,["positive_emo","negative_emo","exclamation","hashtags","question"]): ef_grouped = data_model[data_model[what]>=1].groupby(["label"]).count() plots_data_ef.append({"data":[graph_objs.Bar( x = sentiments, y = [ef_grouped.loc[s,:][0] for s in sentiments], )], "title":"How feature \""+what+"\" separates the tweets"}) for plot_data_ef in plots_data_ef: plotly.offline.iplot({ "data":plot_data_ef["data"], "layout":graph_objs.Layout(title=plot_data_ef["title"]) })

Experiment 2: extended features + Random Forest

As a second attempt on the classification the Random Forest will be used.

from sklearn.ensemble import RandomForestClassifier X_train, X_test, y_train, y_test = train_test_split(data_model.iloc[:, 1:], data_model.iloc[:, 0], train_size=0.7, stratify=data_model.iloc[:, 0], random_state=seed) precision, recall, accuracy, f1 = test_classifier(X_train, y_train, X_test, y_test, RandomForestClassifier(random_state=seed,n_estimators=403,n_jobs=-1))

=============================================== Testing RandomForestClassifier Learing time 6.9144287109375s Predicting time 0.21802711486816406s =================== Results =================== Negative Neutral Positive F1 [ 0.24501425 0.47944007 0.70340909] Precision[ 0.47777778 0.49192101 0.63163265] Recall [ 0.16475096 0.46757679 0.79358974] Accuracy 0.575291948371 ===============================================

The accuracy for the initial split was lower than the one for the Naive Bayes, but let's see what happens during crossvalidation:

rf_acc = cv(RandomForestClassifier(n_estimators=403,n_jobs=-1, random_state=seed),data_model.iloc[:, 1:], data_model.iloc[:, 0])

=============================================== Crossvalidating RandomForestClassifier... Crosvalidation completed in 70.09595036506653s Accuracy: [ 0.54344624 0.50442478 0.37905605 0.27876106 0.37905605 0.52141802 0.5155096 0.55621302] Average accuracy: 0.459735602399 ===============================================

It looks better, however it's still not much above accuracy of the random classifier and barely better than Naive Bayes classifier.

We can observe a low recall level of the RandomForest classifier for the negative class, which may be caused by the data skewness.

More features - word2vec

The overall performance of the previous classifiers could be enhanced by performing time-consuming parameters adjustments, however there's not guarantee on how big the gain will be.

If the out-of-the-shelf methods did not performed well, it seems that there's not much in the data itself. The next idea to add more into data model is to use word2vec representation of a tweet to perform classification.

The word2vec allows to transform words into vectors of numbers. Those vectors represent abstract features, that describe the word similarities and relationships (i.e co-occurence).

What is the best in the word2vec is that operations on the vectors approximately keep the characteristics of the words, so that joining (averaging) vectors from the words from sentence procude vector that is likely to represent the general topic of the sentence.

A lot of pre-trained word2vec models exists, and some of them were trained on huge volumes of data. For the purpose of this analysis, the one trained on over 2 billion of tweets with 200 dimensions (one vector consists of 200 numbers) is used. The pre-trained model can be downloaded here: https://github.com/3Top/word2vec-api

From GloVe to word2vec

In order to use GloVe-trained model in gensim library, it needs to be converted to word2vec format. The only difference between those formats is that word2vec text files starts with two numbers: number of lines in file and number of dim