<jupyter_start><jupyter_text>Question 1 In this question, I am tasked to create a classifier that is able to differentiate Trump and Clinton tweets using only the text in the tweets - we are not allowed to use the location, time and any other information<jupyter_code>from gensim.models.word2vec import Word2Vec
from sklearn.externals import joblib

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from tqdm import tqdm
from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import MinMaxScaler
import sys
import csv
import pandas as pd
import tweepy
import string
from string import maketrans 
import re
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import numpy as np
from sklearn.metrics import accuracy_score
from sklearn.feature_extraction.text import\
TfidfVectorizer, CountVectorizer
from sklearn.model_selection import \
train_test_split, cross_val_score, KFold
from keras.layers import Concatenate
import keras
from keras.models import Model
from keras.models import Sequential
from keras.layers import Dense, Dropout,\
Flatten, Input, Activation, \
Dropout, BatchNormalization, Embedding
from keras.layers import Conv1D, MaxPool1D,\
MaxPooling1D
from keras import backend as K
from keras.models import load_model
from keras import metrics
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
wordnet_lemmatizer = WordNetLemmatizer()
nltk.download('stopwords')
nltk.download('wordnet')<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     /home/adityasidharta/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package wordnet to
[nltk_data]     /home/adityasidharta/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>First, I will use the package `tweepy` in python to download all of the Trump and Hillary tweets. To perform this, we need to create API key using our personal Twitter Account so that we are able to download all the tweets. For privacy reasons, I have deleted my API key.<jupyter_code>ckey = "secret"
csecret = "secret"
atoken = "secret"
asecret = "secret"
auth = tweepy.OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
api = tweepy.API(auth)<jupyter_output><empty_output><jupyter_text>`Tweepy` only allows us to download 200 tweets for a single request. Thus, we will remember the oldest ID of the tweet that we have downloaded, and send multiple request until there are no more new tweets that we can download for both Trump and Hillary. We will then store the ID of the tweet, the creation time of the tweet, as well as the message of the tweet in a csv format. This is so that we can load the csv using `pandas` easily<jupyter_code>def get_all_tweets(screen_name):
    alltweets = []
    new_tweets = api.user_timeline\
    (screen_name = screen_name,count=200)
    alltweets.extend(new_tweets)
    oldest = alltweets[-1].id - 1

    while len(new_tweets) > 0:
        new_tweets = api.user_timeline\
        (screen_name = screen_name,count=200,
         max_id=oldest)
        alltweets.extend(new_tweets)

        oldest = alltweets[-1].id - 1

    outtweets = [[tweet.id_str, 
                  tweet.created_at,
                  tweet.text.encode("utf-8")] \
                 for tweet in alltweets]

    with open('%s_tweets.csv' % screen_name, 
              'wb') as f:
        writer = csv.writer(f)
        writer.writerow(["id","created_at","text"])
        writer.writerows(outtweets)

pass
get_all_tweets('HillaryClinton')
get_all_tweets('RealDonaldTrump')
df_hillary = pd.read_csv('HillaryClinton_tweets.csv')
df_trump = pd.read_csv('realDonaldtrump_tweets.csv')
def text_process(s):
    s = str(s)
    s = s.translate(None, string.punctuation.replace('@',''))
    s = s.translate(None, '1234567890')
    s = [word for word in s.split() if not \
     (word.startswith('http') or word.startswith('@'))]
    s = ' '.join(s)
    s = re.sub(' +',' ',s)
    s = s.decode('ascii', 'ignore')
    s = s.lower()
    s = [word for word in s.split() if word not\
         in stopwords.words('english') + ['rt'] + ['amp'] + ['hillary', 'donald', 'trump' + 'u']]
    s = [wordnet_lemmatizer.lemmatize(word) for word in s]
    sentence = ' '.join(s)
    return sentence
df_hillary['len'] = df_hillary['text'].apply(len)
df_hillary['token'] = df_hillary['text'].apply(lambda x : text_process(x))
df_trump['len'] = df_trump['text'].apply(len)
df_trump['token'] = df_trump['text'].apply(lambda x : text_process(x))
df_hillary['user'] = 'hillary'
df_trump['user'] = 'trump'
df_hillary = df_hillary.loc[df_hillary.token != '', :]
df_trump = df_trump.loc[df_trump.token != '', :]
df_train = pd.concat([df_hillary, df_trump])
df_train
length_hillary = df_train[df_train['user'] == 'hillary'].len.values
length_trump = df_train[df_train['user'] == 'trump'].len.values
fig, axes = plt.subplots(nrows = 2, sharex= True)
sns.distplot(length_hillary, ax= axes[0]).\
set_title('Length of hillary messages')
sns.distplot(length_trump, ax = axes[1]).\
set_title('Length of trump messages')
plt.tight_layout()
plt.show()
hillary_words =  ''
trump_words = ''
array_label = df_train['user'].values
df_messages = df_train['token'].values

for idx in range(len(df_messages)):
    if array_label[idx] == 'hillary':
        hillary_words += '' + str(df_messages[idx])
    else:
        trump_words += '' + str(df_messages[idx])
from wordcloud import WordCloud
hillary_wordcloud = WordCloud(width=600, height=400).generate(hillary_words)
trump_wordcloud = WordCloud(width=600, height=400).generate(trump_words)
plt.figure( figsize=(6,5), facecolor='k')
plt.imshow(hillary_wordcloud)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()
plt.figure( figsize=(6,5), facecolor='k')
plt.imshow(trump_wordcloud)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()
Y_train = np.where(df_train.user == 'hillary', 1, 0)
message = df_train.token.values
kfold = KFold(n_splits = 5, shuffle = True, random_state=1234)
from sklearn.linear_model import LogisticRegressionCV
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.metrics import accuracy_score<jupyter_output><empty_output><jupyter_text>Count<jupyter_code>count_vectorizer = CountVectorizer()
X_train = count_vectorizer.fit_transform(message)
print Y_train.shape
print X_train.shape
log_count_model = LogisticRegressionCV()
log_count_score = cross_val_score(log_count_model,
                                  X_train, Y_train,
                                  scoring = 'accuracy',
                                  cv = kfold.split(X_train),
                                  n_jobs=-1).mean()
print log_count_score
svc_count_model = LinearSVC()
svc_count_score = cross_val_score(svc_count_model,
                                  X_train, Y_train,
                                  scoring = 'accuracy',
                                  cv = kfold.split(X_train),
                                  n_jobs=-1).mean()
print svc_count_score
nb_count_model = MultinomialNB()
nb_count_score = cross_val_score(nb_count_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train,),
                                 n_jobs=-1).mean()
print nb_count_score
gb_count_model = GradientBoostingClassifier()
gb_count_score = cross_val_score(gb_count_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train),
                                 n_jobs=-1).mean()
print gb_count_score
rf_count_model = RandomForestClassifier()
rf_count_score = cross_val_score(rf_count_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train),
                                 n_jobs=-1).mean()
print rf_count_score<jupyter_output>0.8454716764195073<jupyter_text>TF-IDF<jupyter_code>tfidf_vectorizer = TfidfVectorizer("english")
X_train = tfidf_vectorizer.fit_transform(message)
log_tfidf_model = LogisticRegressionCV()
log_tfidf_score = cross_val_score(log_tfidf_model,
                                  X_train, Y_train,
                                  scoring = 'accuracy',
                                  cv = kfold.split(X_train),
                                  n_jobs=-1).mean()
print log_tfidf_score
svc_tfidf_model = LinearSVC()
svc_tfidf_score = cross_val_score(svc_tfidf_model,
                                  X_train, Y_train,
                                  scoring = 'accuracy',
                                  cv = kfold.split(X_train),
                                  n_jobs=-1).mean()
print svc_count_score
nb_tfidf_model = MultinomialNB()
nb_tfidf_score = cross_val_score(nb_tfidf_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train,),
                                 n_jobs=-1).mean()
print nb_tfidf_score
gb_tfidf_model = GradientBoostingClassifier()
gb_tfidf_score = cross_val_score(gb_tfidf_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train),
                                 n_jobs=-1).mean()
print gb_tfidf_score
rf_tfidf_model = RandomForestClassifier()
rf_tfidf_score = cross_val_score(rf_tfidf_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train),
                                 n_jobs=-1).mean()
print rf_tfidf_score<jupyter_output>0.8423590780086052<jupyter_text>Word2Vec - Glove<jupyter_code>def w2v_to_features(w2v, list_tokens):
    w2v_keys = w2v.keys()
    n_dims = len(w2v[w2v.keys()[0]])
    result = np.zeros((len(list_tokens), n_dims))
    for idx in tqdm(range(len(list_tokens))):
        array_features = np.array([w2v[x] 
                                   for x in 
                                   list_tokens[idx] 
                                   if x in w2v_keys])
        if len(array_features) == 0:
            continue
        else:
            result[idx] = array_features.mean(axis = 0)
    return result.astype('float64')
with open('glove.twitter.27B.200d.txt', "rb") as lines:
    w2v = {line.split()[0]: 
                      np.array(map(float, line.split()[1:]))
               for line in lines}
list_tokens = []
for idx in range(len(message)):
    list_tokens.append([str(x) for x in message[idx].split(' ')])
scaler = MinMaxScaler()
X_train = w2v_to_features(w2v, list_tokens)
X_train = scaler.fit_transform(X_train)
joblib.dump(X_train, 'X_train_glove.pkl')
log_glove_model = LogisticRegressionCV()
log_glove_score = cross_val_score(log_glove_model,
                                  X_train, Y_train,
                                  scoring = 'accuracy',
                                  cv = kfold.split(X_train),
                                  n_jobs=-1).mean()
print log_glove_score
svc_glove_model = LinearSVC()
svc_glove_score = cross_val_score(svc_glove_model,
                                  X_train, Y_train,
                                  scoring = 'accuracy',
                                  cv = kfold.split(X_train),
                                  n_jobs=-1).mean()
print svc_glove_score
nb_glove_model = MultinomialNB()
nb_glove_score = cross_val_score(nb_glove_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train,),
                                 n_jobs=-1).mean()
print nb_glove_score
gb_glove_model = GradientBoostingClassifier()
gb_glove_score = cross_val_score(gb_glove_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train),
                                 n_jobs=-1).mean()
print gb_glove_score
rf_glove_model = RandomForestClassifier()
rf_glove_score = cross_val_score(rf_glove_model, 
                                 X_train, Y_train,
                                 scoring = 'accuracy', 
                                 cv = kfold.split(X_train),
                                 n_jobs=-1).mean()
print rf_glove_score<jupyter_output>0.7262708243823033<jupyter_text>Embedding - Dense<jupyter_code>tokenizer = Tokenizer()
tokenizer.fit_on_texts(message)
sequences = tokenizer.texts_to_sequences(message)
length = []
for x in sequences:
    length.extend(x)
max(length)
X_train = pad_sequences(sequences, maxlen=21)
X_train.shape
inputs = Input(shape=(21,))
X = Embedding(input_dim = 8555, output_dim = 100, input_length=21)(inputs)
X = Flatten()(X)
X = Dense(100, activation='tanh')(X)
X = BatchNormalization()(X)
X = Dropout(0.05)(X)
X = Dense(1, activation='sigmoid')(X)

FCN_model = Model(inputs, X, name='FCN_model')
FCN_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
FCN_model.summary()
FCN_model.save_weights('FCN_model.h5')
kfold_score = []
for fit_index, val_index in kfold.split(X_train):
    FCN_model.load_weights('FCN_model.h5')
    X_fit, X_val = X_train[fit_index], X_train[val_index]
    Y_fit, Y_val = Y_train[fit_index], Y_train[val_index]
    FCN_model.fit(X_fit, Y_fit, batch_size=32, epochs = 3)
    Y_val_pred = FCN_model.predict(X_val, batch_size=32).reshape(-1)
    Y_val_pred = np.where(Y_val_pred >= 0.50, 1, 0)
    kfold_score.append(accuracy_score(Y_val, Y_val_pred))
FCN_score = np.mean(kfold_score)
FCN_score<jupyter_output><empty_output><jupyter_text>Embedding - Conv<jupyter_code>inputs = Input(shape=(21,), name = 'Input')
X = Embedding(input_dim = 8555, output_dim = 100, input_length=21, name = 'Embedding')(inputs)
X = Conv1D(100, kernel_size=2, padding='valid', activation='tanh', strides=1, name = 'Conv1')(X)
X = BatchNormalization(name = 'BN1')(X)
X = Dropout(0.10, name = 'Drop1')(X)
X = MaxPool1D(2, 2, name = 'MaxPool1')(X)

X = Conv1D(100, kernel_size=2, padding='same', activation='tanh', strides=1, name = 'Conv2')(X)
X = BatchNormalization(name = 'BN2')(X)
X = Dropout(0.10, name = 'Drop2')(X)
X = MaxPool1D(2, 2, name = 'MaxPool1')(X)

X = Flatten()(X)
X = Dense(100, activation='tanh', name = '')(X)
X = BatchNormalization()(X)
X = Dense(1, activation='sigmoid')(X)

CCN_model = Model(inputs, X, name='CCN_model')
CCN_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
CCN_model.summary()
CCN_model.save_weights('CCN_model.h5')
kfold_score = []
for fit_index, val_index in kfold.split(X_train):
    CCN_model.load_weights('CCN_model.h5')
    X_fit, X_val = X_train[fit_index], X_train[val_index]
    Y_fit, Y_val = Y_train[fit_index], Y_train[val_index]
    CCN_model.fit(X_fit, Y_fit, batch_size=32, epochs = 3)
    Y_val_pred = CCN_model.predict(X_val, batch_size=32).reshape(-1)
    Y_val_pred = np.where(Y_val_pred >= 0.50, 1, 0)
    kfold_score.append(accuracy_score(Y_val, Y_val_pred))
CCN_score = np.mean(kfold_score)
CCN_score<jupyter_output><empty_output><jupyter_text>Embedding - Inception-CNN<jupyter_code>inputs = Input(shape=(21,))

X = Embedding(input_dim = 8555, output_dim = 100, input_length=21)(inputs)

bigram = Conv1D(50, kernel_size=2, padding='same', activation='tanh', strides=1)(X)
bigram = BatchNormalization()(bigram)
bigram = Dropout(0.10)(bigram)
bigram = MaxPooling1D(2, 2)(bigram)

trigram = Conv1D(30, kernel_size=3, padding='same', activation='tanh', strides=1)(X)
trigram = BatchNormalization()(trigram)
trigram = Dropout(0.10)(trigram)
trigram = MaxPooling1D(2, 2)(trigram)

quadgram = Conv1D(20, kernel_size=4, padding='same', activation='tanh', strides=1)(X)
quadgram = BatchNormalization()(quadgram)
quadgram = Dropout(0.10)(quadgram)
quadgram = MaxPooling1D(2, 2)(quadgram)

X = Concatenate(axis=-1)([bigram, trigram, quadgram])
X = Flatten()(X)
X = Dense(200, activation='tanh')(X)
X = BatchNormalization()(X)
X = Dense(1, activation='sigmoid')(X)

Incep_CCN_model = Model(inputs, X, name='Incep_CCN_model')
Incep_CCN_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
Incep_CCN_model.summary()
Incep_CCN_model.save_weights('Incep_CCN_model.h5')
kfold_score = []
for fit_index, val_index in kfold.split(X_train):
    Incep_CCN_model.load_weights('Incep_CCN_model.h5')
    X_fit, X_val = X_train[fit_index], X_train[val_index]
    Y_fit, Y_val = Y_train[fit_index], Y_train[val_index]
    Incep_CCN_model.fit(X_fit, Y_fit, batch_size=32, epochs = 3)
    Y_val_pred = Incep_CCN_model.predict(X_val, batch_size=32).reshape(-1)
    Y_val_pred = np.where(Y_val_pred >= 0.50, 1, 0)
    kfold_score.append(accuracy_score(Y_val, Y_val_pred))
Incep_CCN_score = np.mean(kfold_score)
Incep_CCN_score
result = {
    'label' : ['log_count_score' , 'svc_count_score',
               'nb_count_score', 'gb_count_score',
              'rf_count_score', 'log_tfidf_score',
              'svc_tfidf_score', 'nb_tfidf_score',
              'gb_tfidf_score', 'rf_tfidf_score',
              'log_glove_score', 'svc_glove_score',
              'nb_glove_score', 'gb_glove_score',
              'rf_glove_score', 'FCN_score',
              'CCN_score', 'Incep_CCN_score'],
    'score' : [log_count_score , svc_count_score,
               nb_count_score, gb_count_score,
              rf_count_score, log_tfidf_score,
              svc_tfidf_score, nb_tfidf_score,
              gb_tfidf_score, rf_tfidf_score,
              log_glove_score, svc_glove_score,
              nb_glove_score, gb_glove_score,
              rf_glove_score, FCN_score,
              CCN_score, Incep_CCN_score]
}

df_result = pd.DataFrame(result)
display(df_result.sort_values('score', ascending=False))
import seaborn as sns
df_result = df_result.sort_values('score', ascending=False)
f, ax = plt.subplots(figsize=(15, 6))
sns.set_color_codes("pastel")
ax = sns.barplot(x='score', y='label', data=df_result, orient='h')
sns.despine(left=True, bottom=True)
final_model = MultinomialNB().fit(X_train, Y_train)
from IPython.display import Image 

def TrumporClinton(tweet):
    processed_tweet =  text_process(tweet)
    X_test = count_vectorizer.transform(np.array([processed_tweet]))
    if final_model.predict(X_test)[0] == 0:
        return Image(filename='trump.jpg', width = 100, height =70)
    else:
        return Image(filename='hillary.jpeg', width = 100, height =70)
TrumporClinton("We are going to build a wall and Mexico is going to pay for it!")
TrumporClinton('Make America Great Again! China is Stealing our Job!')
TrumporClinton("We cannot allow Trump to be president. This man is dangerous.")
TrumporClinton("Gay rights are human rights.")
TrumporClinton('<NAME>, the worst FBI Director in history, was not fired because of the phony Russia investigation where, by the way, there was NO COLLUSION (except by the Dems)!')
TrumporClinton('Thinking about <NAME>’s legacy of service to our country and the extraordinary family she raised. Thanking her for her many kindnesses to me and my family. Wishing her the comfort she deserves surrounded by her loved ones.')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1110941475826972, 'fraction_numerical': 0.016234096692111958, 'mean_word_length': 3.0854469854469855, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 73, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '28981', 'license': 'MIT', 'n_tokens_mistral': 7022, 'n_tokens_neox': 6695, 'n_words': 1346, 'path': 'Trump_hillary.ipynb', 'repo_name': 'AdityaSidharta/Classify_trump_hillary_tweets', 'size': '19670'}