<jupyter_start><jupyter_text>Load Data<jupyter_code>train = pd.read_csv("twitter_train.csv")
# print(train.shape) # 31962 x 3
train.head() # label: 1 = racist/sexist; 0 = not racist/sexist
# sample tweets from both labels
np.random.seed(22)
idx0 = np.where(train['label'] == 0)
idx1 = np.where(train['label'] == 1)
print(train['tweet'][np.random.choice(idx0[0],10)]) # non-racist/sexist
print(train['tweet'][np.random.choice(idx1[0],10)]) # racist/sexist<jupyter_output>11966    flaxseed gel/shampoo results!  #peace #love #o...
24368     @user great end to a great day! happy weekend...
31695    @user given what we now know, how the church s...
26812             happy day!  #altwaystoheal  #healthy    
23501    18 km in the rain along the thames. perfect st...
24759    #zaynmalik   bull up: you will dominate your b...
22545                  @user true dat ... thanks !   @user
17153    #fruit #tea #peach #drink #yummy   #dayoff #li...
29593     @user @user well here i am...just waiting...a...
20324    okay i think i've got my @user and @user chann...
Name: tweet, dtype: object
14082    are you #black &amp; feel like the  are â¦on ...
24026    @user after 8 yrs of endless lies, contempt fr...
22285    thought factory: left-right polarisation! #tru...
10852    @user #allahsoil not all muslims hate america....
14530    trump is a cuck  @user #nazi #swastika #ebay! ...
16054    @user comment on @user reveals once more the d...
13425    sign that i checked [...]<jupyter_text>Sample + Format Data<jupyter_code>def list_to_onehot(y, list_classes):
    """
    y = list of class lables of length n
    output = n x k array, i th row = one-hot encoding of y[i] (e.g., [0,0,1,0,0])
    """
    Y = np.zeros(shape = [len(y), len(list_classes)], dtype=int)
    for i in np.arange(Y.shape[0]):
        for j in np.arange(len(list_classes)):
            if y[i] == list_classes[j]:
                Y[i,j] = 1
    return Y
def onehot_to_list(y, list_classes=None):
    """
    y = n x k array, i th row = one-hot encoding of y[i] (e.g., [0,0,1,0,0])
    output =  list of class lables of length n
    """
    if list_classes is None:
        list_classes = np.arange(y.shape[1])
        
    y_list = []
    for i in np.arange(y.shape[0]):
        idx = np.where(y[i,:]==1)
        idx = idx[0][0]
        y_list.append(list_classes[idx])
    return y_list
def sample_tweets(list_classes=[0, 1], categories=[0,1], train_size=0.8, min_df = 2, vectorizer = 'tf-idf', full_data=None,):
    """
    list_classes = one-hot labels for each class
    train_size = proportion of full_data to split to train
    min_df = minimum word frequency for vectorizer
    vectorizer = either count or tf-idf
    output = train and test X and Y, data_dict
    """
    
    # remove stopwords and numbers from tweets
    stopwords_list = stopwords.words('english')
    stopwords_list.extend(["’", "“", "”", "https", "co", "user", "amp", "get", "go", "one"]) # remove stopwords
    train = [re.sub(r'\d+','', tweet) for tweet in full_data.tweet] # remove numbers from tweets
    y = full_data.label # tweet class labels 
    Y = list_to_onehot(y.tolist(), list_classes)

    # set vectorizer
    # min-df = n; requires words to appear at least n times in corpus
    if vectorizer == 'tf-idf':
        vectorizer = TfidfVectorizer(stop_words=stopwords_list,min_df=2)
    else:
        vectorizer = CountVectorizer(stop_words=stopwords_list,min_df=2) 
    X = vectorizer.fit_transform(train) # convert to words x docs (word frequency)
    X = np.asarray(X.todense())
    print('X.shape:', X.shape)
    idx_to_word = np.array(vectorizer.get_feature_names())   # list of words that corresponds to feature coordinates

    # update data dict
    data_dict = {}
    data_dict.update({'categories': categories})
    data_dict.update({'train': train})
    data_dict.update({'tweet_labels': y})
    data_dict.update({'feature_matrix': X})
    data_dict.update({'idx_to_word': idx_to_word})

    # train-test split
    X_train = []
    X_test = []
    y_test = []
    y_train = []

    for i in np.arange(X.shape[0]):
        # for each example i, make it into train set with probabiliy 0.8 and into test set otherwise 
        U = np.random.rand() # Uniform([0,1]) variable
        if U < train_size:
            X_train.append(X[i,:])
            y_train.append(Y[i,:].copy())
        else:
            X_test.append(X[i,:])
            y_test.append(Y[i,:].copy())

    X_train = np.asarray(X_train)
    X_test = np.asarray(X_test)
    y_train = np.asarray(y_train)
    y_test = np.asarray(y_test)
    
    data_dict.update({'X_train': X_train})
    data_dict.update({'X_test': X_test})
    data_dict.update({'y_train': y_train})
    data_dict.update({'y_test': y_test})
  
    return X_train, X_test, y_train, y_test, data_dict 

categories = ['non-racist/sexist','racist/sexist'] 

# fit with tf-idf vectorizer
X_train, X_test, y_train, y_test, data_dict = sample_tweets(list_classes=[0, 1],
                                                            categories = categories,
                                                            train_size = 0.8,
                                                            min_df = 2,
                                                            vectorizer = 'tf-idf',
                                                            full_data=train)
print('X_train.shape:', X_train.shape)
print('X_test.shape:', X_test.shape)
print('y_train.shape:', y_train.shape)
print('y_test.shape:', y_test.shape)
print('y_test\n', y_test)
print('y_list\n', onehot_to_list(y_test)[:6])
print('idx_to_word\n', data_dict['idx_to_word'])<jupyter_output>X.shape: (31962, 14896)
X_train.shape: (25556, 14896)
X_test.shape: (6406, 14896)
y_train.shape: (25556, 2)
y_test.shape: (6406, 2)
y_test
 [[1 0]
 [1 0]
 [1 0]
 ...
 [1 0]
 [0 1]
 [1 0]]
y_list
 [0, 0, 0, 0, 0, 1]
idx_to_word
 ['_animaladvocate' '_pearls' '_ë³' ... 'ð¾ñ' 'ó¾' 'ø¹ù']<jupyter_text>Check most frequent words for other stopwords to remove<jupyter_code>idx_to_word = data_dict['idx_to_word']
sums = np.sum(X_train,axis=0)
word_freq = pd.DataFrame(data={'word': idx_to_word, 'frequency (tf-idf)': sums})
word_freq.sort_values(by=['frequency (tf-idf)'],ascending=False)[:20]<jupyter_output><empty_output><jupyter_text>removed words: user, amp (ampersand?), get, go, one  Wordcloud for each label in train set<jupyter_code># word cloud for different labels
def plot_wordcloud(Y, H, idx_to_word, categories):
        # plot the class-conditioanl PMF as wordclouds 
        # Y = (n x k), H = (p x n) (\Phi in lecture note), W = (p x k)
        # idx_to_word = list of words used in the vectorization of documents 
        # categories = list of class labels
        # prior on class labels = empirical PMF = [ # class i examples / total ]
        # class-conditional for class i = [ # word j in class i examples / # words in class i examples]
        
        class_conditional_PMF = []
        for i in np.arange(Y.shape[1]):
            idx = np.where(Y[:,i]==1)
            sub_H = H[:,idx[0]]
            word_count_per_class = np.sum(sub_H, axis=1)
            class_conditional_PMF.append(word_count_per_class/np.sum(word_count_per_class))
    
        ### topic mode
        num_words_to_sample = 10000
        
        fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(14, 7), subplot_kw={'xticks': [], 'yticks': []})
        for ax, i in zip(axs.flat, np.arange(Y.shape[0])):
            dist = class_conditional_PMF[i]

            # sample (num_words_to_sample) proportionate to CC_PMF
            list_words = []
        
            for j in range(num_words_to_sample):
                idx = np.random.choice(np.arange(H.shape[0]), p=dist)
                list_words.append(idx_to_word[idx])
                
            Y = " ".join(list_words) # list to string
#             print(Y)
            STOPWORDS.update(["’", "“", "”", "https", "co", "user", "amp", "get", "go", "one"])
            wordcloud = WordCloud(background_color="black",
                                  relative_scaling=0.1,
                                  width=400,
                                  height=400).generate(Y)
            ax.imshow(wordcloud, interpolation='bilinear')
            ax.set_xlabel(categories[i], fontsize='20')
            # ax.axis("off")

        plt.tight_layout()
        plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.08)
        plt.savefig('wordcloud_tf-idf_twitter.pdf', bbox_inches='tight')

plot_wordcloud(Y=np.asarray(y_train), H=X_train.T, idx_to_word=idx_to_word, categories=categories)<jupyter_output><empty_output><jupyter_text>Naive Bayes Classifier<jupyter_code>def fit_NB(Y, H,pseudocount = 10**-5):
    '''
    Fit Multinomial Naive Bayes Calssifier
    Use the Maximum Likelihood prior and class conditional probabilities (in closed forms)
    Y = (n x k), H = (p x n) (\Phi in lecture note), W = (p x k)
    prior on class labels = empirical PMF = [ # class i examples / total ]
    class-conditional for class i = [ # word j in class i examples / # words in class i examples]
    Output = prior (k, ), class_conditional_PMF = (k, p)
    '''
    k = Y.shape[1] # number of classes (2)

    prior = np.sum(Y, axis=0)/np.sum(np.sum(Y, axis=0))
#     print(prior)

    class_conditional_PMF = []
    for i in np.arange(Y.shape[1]):
        idx = np.where(Y[:,i]==1) # select indices with class 0 or 1
        # sub_H: documents with matching class
        sub_H = H[:,idx[0]] + pseudocount # add psuedocount
        word_count_per_class = np.sum(sub_H, axis=1)
        class_conditional_PMF.append(word_count_per_class/np.sum(word_count_per_class))

    return prior, np.asarray(class_conditional_PMF)

# find prior and class_conditional of train data 
prior, class_conditional_PMF = fit_NB(Y=y_train, H=X_train.T)
print(prior)
print(class_conditional_PMF)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=[15, 5])
for i in [0,1]:
    ax.plot(np.arange(X_train.shape[1]), class_conditional_PMF[i], label=categories[i])
ax.legend(fontsize='15')
plt.suptitle("Class conditional PMFs (tf-idf)", fontsize='15')
plt.savefig('PMF_dist_tf-idf_twitter.pdf', bbox_inches='tight')
def predict_MNB(X_test, prior, class_conditional_PMF):
    '''
    Compute predicted PMF for the test data given prior and class_conditional_PMF
    Simple use of Bayes' Theorem 
    X_test = (p x n) (words x docs)
    '''
    print(X_test.shape)
    print(class_conditional_PMF.shape)
    
    P = class_conditional_PMF / np.min(class_conditional_PMF) # normalize so that log(P) is not too small 
    Q = np.exp(X_test @ np.log(P).T)
    Q = Q * np.repeat(prior[:, np.newaxis], repeats=Q.shape[0], axis=1).T
    sum_of_rows = Q.sum(axis=1)
    
    return Q / sum_of_rows[:, np.newaxis]

# find predictive PMF for X_test
predictive_PMF = predict_MNB(X_test, prior, class_conditional_PMF)
print(predictive_PMF[:6])
plt.title("Tweet 5")
plt.plot(predictive_PMF[4])
def compute_accuracy_metrics(Y_test, P_pred, use_opt_threshold=False, verbose=True):
    
    # y_test = binary label 
    # P_pred = predicted probability for y_test
    # compuate various binary classification accuracy metrics
    fpr, tpr, thresholds = metrics.roc_curve(Y_test, P_pred, pos_label=None)
    mythre = thresholds[np.argmax(tpr - fpr)]
    myauc = metrics.auc(fpr, tpr)
    # print('!!! auc', myauc)
    
    # Compute classification statistics
    threshold = 0.5
    if use_opt_threshold:
        threshold = mythre
    
    Y_pred = P_pred.copy()
    Y_pred[Y_pred < threshold] = 0
    Y_pred[Y_pred >= threshold] = 1

    mcm = confusion_matrix(Y_test, Y_pred)
    
    tn = mcm[0, 0]
    tp = mcm[1, 1]
    fn = mcm[1, 0]
    fp = mcm[0, 1]

    accuracy = (tp + tn) / (tp + tn + fp + fn)
    sensitivity = tn / (tn + fp)
    specificity = tp / (tp + fn)
    precision = tp / (tp + fp)
    fall_out = fp / (fp + tn)
    miss_rate = fn / (fn + tp)

    # Save results
    results_dict = {}
    results_dict.update({'Y_test': Y_test})
    results_dict.update({'Y_pred': Y_pred})
    results_dict.update({'AUC': myauc})
    results_dict.update({'Opt_threshold': mythre})
    results_dict.update({'Accuracy': accuracy})
    results_dict.update({'Sensitivity': sensitivity})
    results_dict.update({'Specificity': specificity})
    results_dict.update({'Precision': precision})
    results_dict.update({'Fall_out': fall_out})
    results_dict.update({'Miss_rate': miss_rate})
    results_dict.update({'Confusion_mx': mcm})
    
    
    if verbose:
        for key in [key for key in results_dict.keys()]:
            if key not in ['Y_test', 'Y_pred', 'Confusion_mx']:
                print('% s ===> %.3f' % (key, results_dict.get(key)))
#         print('Confusion matrix \n ===>', mcm)
            
    return results_dict<jupyter_output><empty_output><jupyter_text>Check accuracy of tf-idf vectorizer<jupyter_code>result_dict_list = []
W_list = []
list_vectorizer = ['tf-idf']
for i in np.arange(len(list_vectorizer)):
    # compute metrics
    results_dict = compute_accuracy_metrics(Y_test=onehot_to_list(y_test), P_pred=predictive_PMF[:,1], verbose=True)
    result_dict_list.append(results_dict.copy())
    W_list.append(results_dict.get('Confusion_mx').copy())

    # scale W_list by count to format as proportion for confusion matrix
    true_counts = np.sum(W_list[0],axis=0)
    W_list2 = []
    conf_mat_prop = W_list[0] / true_counts
    print('Confusion Matrix ===>\n', conf_mat_prop)
    W_list2.append(conf_mat_prop)
# make plot
fig, ax = plt.subplots(nrows=1, ncols=len(list_vectorizer), figsize=[12, 7])
class_labels= categories
for i in np.arange(len(list_vectorizer)):
    results_dict = result_dict_list[i]
    confusion_mx = results_dict.get('Confusion_mx')
    if len(list_vectorizer) > 1:
        confusion_mx = W_list2[i]
        im = ax[i].imshow(confusion_mx, vmin=np.min(W_list2), vmax=np.max(W_list2))
        ax[i].title.set_text("NB Confusion mx using %s vectorizer" % list_vectorizer[i])
        # ax[i].legend()
        # if i == len(list_digits) - 1:
        ax[i].set_xticks(np.arange(len(class_labels)))
        # ax[i].set_xticklabels(class_labels)
        ax[i].set_yticks(np.arange(len(class_labels)))
        ax[i].set_yticklabels(class_labels)
    else:
        confusion_mx = W_list2[i]
        im = ax.imshow(confusion_mx, vmin=np.min(W_list2), vmax=np.max(W_list2))
        ax.title.set_text("NB Confusion mx using %s vectorizer" % list_vectorizer[i])
        # ax[i].legend()
        # if i == len(list_digits) - 1:
        ax.set_xticks(np.arange(len(class_labels)))
        ax.set_yticks(np.arange(len(class_labels)))
        ax.set_yticklabels(class_labels, fontsize='15')
    
    subtitle = "vectorizer: " + str(list_vectorizer[i]) + "\n" + "classifier: " + str('Naive Bayes')
    keys_list = [i for i in results_dict.keys()]
    for key in keys_list:
        if key not in ['Y_test', 'Y_pred', 'AUC', 'Opt_threshold', 'Confusion_mx']:
            subtitle += "\n" + str(key) + " = " + str(np.round(results_dict.get(key),3))
            # print('%s = %f' % (key, results_list[i].get(key)))
    ax.set_title(subtitle, fontsize='15')
    
cbar_ax = fig.add_axes([0.8, 0.15, 0.01, 0.7])
fig.colorbar(im, cax=cbar_ax)
plt.savefig('confusion_mat_tf-idf_twitter.pdf', bbox_inches='tight')
# try model on trump tweets
trump = pd.read_csv("trump_tweets.csv")
trump.head()
 # remove stopwords and numbers from tweets
trump_x = [re.sub(r'\d+','', tweet) for tweet in trump.text] # remove numbers from tweets
stopwords_list = stopwords.words('english')
stopwords_list.extend(["’", "“", "”", "https", "co", "user", "amp", "get", "go", "one","realdonaldtrump","rt",
                      "http","would","make","time"])
    
# set vectorizer
# min-df = n; requires words to appear at least n times in corpus
vectorizer = TfidfVectorizer(stop_words=stopwords_list,min_df=2)

# fit vectorizer
trump_X = vectorizer.fit_transform(trump_x) # convert to words x docs (word frequency)
trump_X = np.asarray(trump_X.todense())
print('trump_X.shape:', trump_X.shape)
idx_to_word = np.array(vectorizer.get_feature_names())   # list of words that corresponds to feature coordinates
# most common words
sums = np.sum(trump_X,axis=0)
word_freq = pd.DataFrame(data={'word': idx_to_word, 'frequency (tf-idf)': sums})
word_freq.sort_values(by=['frequency (tf-idf)'],ascending=False)[:20]
# fit NB
prior, class_conditional_PMF = fit_NB(Y=y_train, H=trump_X.T)
print(prior)
print(class_conditional_PMF)
# get predictive probabilities
predictive_PMF = predict_MNB(trump_X, prior, class_conditional_PMF)
print(predictive_PMF[:6])
plt.plot(predictive_PMF[4])
# classify with threshold of 0.5
threshold = 0.5
Y_pred = predictive_PMF.copy()
Y_pred[Y_pred < threshold] = 0
Y_pred[Y_pred >= threshold] = 1
Y_pred[:6]
# 99.5% of trumps tweets are classified as non-racist/sexist
trump_sums = np.sum(Y_pred,axis=0)
print(trump_sums/np.sum(trump_sums))
print(prior)<jupyter_output>[0.99540401 0.00459599]
[0.93030991 0.06969009]
{'fraction_non_alphanumeric': 0.13946919319719886, 'fraction_numerical': 0.022538692402754076, 'mean_word_length': 3.132782101167315, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '203012', 'license': 'MIT', 'n_tokens_mistral': 6394, 'n_tokens_neox': 6065, 'n_words': 1382, 'path': 'notebooks/M156 Final Project NB tf_idf.ipynb', 'repo_name': 'tylerwu2222/Math156_UCLA_SP21', 'size': '16993'}