<jupyter_start><jupyter_text>Set Up 导入需要的模块<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import nltk
from nltk.tokenize import WordPunctTokenizer

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn import svm
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.model_selection import KFold

import torch
import torch.nn as nn
from torch.autograd import Variable

import stats<jupyter_output><empty_output><jupyter_text>读取训练集数据和测试集数据并定义分词器<jupyter_code>TRAIN_FILE = '../data/processed_train.csv'
TEST_FILE = '../data/processed_test.csv'

train_data = pd.read_csv(TRAIN_FILE)
test_data = pd.read_csv(TEST_FILE)
word_tokenizer = WordPunctTokenizer()

def bigram_tokenize(text):
    bigrams = []
    unigrams = word_tokenizer.tokenize(text)
    for i in range(len(unigrams) - 1):
        bigrams.append('-'.join([unigrams[i], unigrams[i+1]]))
    return bigrams<jupyter_output><empty_output><jupyter_text>简单查看分词情况<jupyter_code># count totally how many unique word in the whole train dataset and test dataset
print('TRAIN')
stats.displayStatsInfo(TRAIN_FILE)

print('\nTEST')
stats.displayStatsInfo(TEST_FILE)<jupyter_output>TRAIN

Analysis Statistics
Tweets => Total: 11712, Positive: 1919, Neutral: 2514, Negative: 7279
User Mentions => Total: 13246, Avg: 1.1310, Max: 6
URLs => Total: 972, Avg: 0.0830, Max: 3
Emojis => Total: 291, Positive: 204, Negative: 87, Avg: 0.0248, Max: 3
Unigrams => Total: 11712, Unique: 11594, Avg: 20.8228, Max: 47, Min: 2
Bigrams => Total: 232165, Unique: 84517, Avg: 19.8228

TEST

Analysis Statistics
Tweets => Total: 2928, Positive: 444, Neutral: 585, Negative: 1899
User Mentions => Total: 3259, Avg: 1.1130, Max: 5
URLs => Total: 239, Avg: 0.0816, Max: 2
Emojis => Total: 60, Positive: 45, Negative: 15, Avg: 0.0205, Max: 2
Unigrams => Total: 2928, Unique: 5564, Avg: 20.9822, Max: 46, Min: 2
Bigrams => Total: 58508, Unique: 30751, Avg: 19.9822<jupyter_text>定义训练集和测试集的标签/目标<jupyter_code>train_targets = train_data.sentiment
test_targets = test_data.sentiment<jupyter_output><empty_output><jupyter_text>Naive Bayes 文本特征提取：词频向量化 + TF-IDF 处理<jupyter_code>countVzer = CountVectorizer(min_df=7, max_df=0.5, tokenizer=word_tokenizer.tokenize)
train_counts = countVzer.fit_transform(train_data.text)
test_counts = countVzer.transform(test_data.text)
tfidfTfmer = TfidfTransformer(smooth_idf=True, sublinear_tf=True, use_idf=True)
train_tfidfs = tfidfTfmer.fit_transform(train_counts)
test_tfidfs = tfidfTfmer.transform(test_counts)
print('feature dimension: {}'.format(train_counts.shape[1]))
clf_NB = MultinomialNB(alpha=1.0e-10)
clf_NB.fit(train_counts, train_targets)
test_predicts = clf_NB.predict(test_counts)
print('Navie Bayes without Laplace Smoothing')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))
clf_NB = MultinomialNB(alpha=1.0)
clf_NB.fit(train_counts, train_targets)
test_predicts = clf_NB.predict(test_counts)
print('Navie Bayes with Laplace Smoothing')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))
clf_NB = MultinomialNB(alpha=1.0e-10)
clf_NB.fit(train_tfidfs, train_targets)
test_predicts = clf_NB.predict(test_tfidfs)
print('Navie Bayes without Laplace Smoothing (apply TF-IDF)')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))
clf_NB = MultinomialNB(alpha=1.0)
clf_NB.fit(train_tfidfs, train_targets)
test_predicts = clf_NB.predict(test_tfidfs)
print('Navie Bayes with Laplace Smoothing (apply TF-IDF)')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))<jupyter_output>Navie Bayes with Laplace Smoothing (apply TF-IDF)
              precision    recall  f1-score   support

    negative       0.74      0.98      0.85      1899
     neutral       0.75      0.24      0.36       585
    positive       0.86      0.44      0.58       444

   micro avg       0.75      0.75      0.75      2928
   macro avg       0.78      0.55      0.59      2928
weighted avg       0.76      0.75      0.71      2928

Accuracy: 0.7513661202185792<jupyter_text>准确率最高大概在 0.77~0.78  SVM<jupyter_code>countVzer = CountVectorizer(tokenizer=word_tokenizer.tokenize)
train_counts = countVzer.fit_transform(train_data.text)
test_counts = countVzer.transform(test_data.text)
tfidfTfmer = TfidfTransformer(smooth_idf=True, sublinear_tf=True, use_idf=True)
train_tfidfs = tfidfTfmer.fit_transform(train_counts)
test_tfidfs = tfidfTfmer.transform(test_counts)
print('feature dimension: {}'.format(train_counts.shape[1]))
train_counts[:int(train_counts.shape[0]*0.7)]
train_counts[int(train_counts.shape[0]*0.7):]
clf_SVM = svm.LinearSVC(C=0.03)
clf_SVM.fit(train_counts, train_targets)
test_predicts = clf_SVM.predict(test_counts)
print('SVM with linear kernel')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))
output = pd.DataFrame(data={'id':range(1, len(test_predicts)+1), 'polarity': test_predicts})
output.to_csv("./result/svm-not-apply-tfidf.csv", index=False)
clf_SVM = svm.LinearSVC(C=0.28)
clf_SVM.fit(train_tfidfs, train_targets)
test_predicts = clf_SVM.predict(test_tfidfs)
print('SVM with linear kernel (apply TF-IDF)')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))
output = pd.DataFrame(data={'id':range(1, len(test_predicts)+1), 'polarity': test_predicts})
output.to_csv("./result/svm-apply-tfidf.csv", index=False)<jupyter_output>SVM with linear kernel (apply TF-IDF)
              precision    recall  f1-score   support

    negative       0.84      0.94      0.89      1899
     neutral       0.69      0.53      0.60       585
    positive       0.80      0.63      0.70       444

   micro avg       0.81      0.81      0.81      2928
   macro avg       0.78      0.70      0.73      2928
weighted avg       0.80      0.81      0.80      2928

Accuracy: 0.8128415300546448<jupyter_text>准确率最高可到 0.81  KNN<jupyter_code>countVzer = CountVectorizer(min_df=0, max_df=0.9, tokenizer=word_tokenizer.tokenize)
train_counts = countVzer.fit_transform(train_data.text)
test_counts = countVzer.transform(test_data.text)
tfidfTfmer = TfidfTransformer(smooth_idf=True, sublinear_tf=True, use_idf=True)
train_tfidfs = tfidfTfmer.fit_transform(train_counts)
test_tfidfs = tfidfTfmer.transform(test_counts)
print('feature dimension: {}'.format(train_counts.shape[1]))
from sklearn import neighbors

clf_KNN = neighbors.KNeighborsClassifier(n_neighbors=2)
clf_KNN.fit(train_counts, train_targets)
test_predicts = clf_KNN.predict(test_counts)
print('KNN-2')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))
clf_KNN = neighbors.KNeighborsClassifier(n_neighbors=35)
clf_KNN.fit(train_tfidfs, train_targets)
test_predicts = clf_KNN.predict(test_tfidfs)
print('KNN-35')
print(classification_report(test_targets, test_predicts))
print('Accuracy: {}'.format(accuracy_score(test_targets, test_predicts)))<jupyter_output>KNN-35
              precision    recall  f1-score   support

    negative       0.77      0.94      0.84      1899
     neutral       0.65      0.25      0.36       585
    positive       0.72      0.60      0.66       444

   micro avg       0.75      0.75      0.75      2928
   macro avg       0.71      0.60      0.62      2928
weighted avg       0.74      0.75      0.72      2928

Accuracy: 0.7513661202185792<jupyter_text>准确率 0.75 左右  FM<jupyter_code>countVzer = CountVectorizer(min_df=0, max_df=0.9, tokenizer=word_tokenizer.tokenize)
train_counts = countVzer.fit_transform(train_data.text)
test_counts = countVzer.transform(test_data.text)
tfidfTfmer = TfidfTransformer(smooth_idf=True, sublinear_tf=True, use_idf=True)
train_tfidfs = tfidfTfmer.fit_transform(train_counts)
test_tfidfs = tfidfTfmer.transform(test_counts)
print('feature dimension: {}'.format(train_counts.shape[1]))
class FM(nn.Module):
    def __init__(self, n=10, k=5):
        super(FM, self).__init__()
        self.n = n
        self.k = k
        self.linear = nn.Linear(self.n, 3)   # 前两项线性层
        self.V = nn.Parameter(torch.randn(self.n, self.k))   # 交互矩阵
    def fm_layer(self, x):
        linear_part = self.linear(x)
        interaction_part_1 = torch.mm(x, self.V)
        interaction_part_1 = torch.pow(interaction_part_1, 2)
        interaction_part_2 = torch.mm(torch.pow(x, 2), torch.pow(self.V, 2))
        output = linear_part + torch.sum(0.5 * interaction_part_2 - interaction_part_1)
        
        return output
    def forward(self, x):
        return self.fm_layer(x)
num_epochs = 7
batch_size = 64
lr = 5e-4
weight_decay = 1e-5
k = 100

device = torch.device("cuda:1")

clf_FM = FM(train_counts.shape[1], k)
clf_FM.to(device)
optimizer = torch.optim.Adam(clf_FM.parameters(), lr=lr, weight_decay=weight_decay)
loss_fn = nn.CrossEntropyLoss()
kf = KFold(n_splits=5, shuffle=True, random_state=0)

fm_train_losses = []
fm_train_accs = []
fm_test_losses = []
fm_test_accs = []

for f, (train_index, val_index) in enumerate(kf.split(train_counts)):
    print('Fold[{}]'.format(f+1))
    for epoch in range(num_epochs):
        clf_FM.train()
        fm_loss = 0
        fm_acc = 0
        for i in range(int(len(train_index)/batch_size)):
            batch_train_counts = Variable(torch.FloatTensor(
                train_counts[train_index][i*batch_size:i*batch_size+batch_size].toarray())).to(device)
            batch_train_targets = Variable(torch.LongTensor(
                train_targets[train_index][i*batch_size:i*batch_size+batch_size].map({
                    'negative': 0, 'neutral': 1, 'positive': 2
                }).values)).to(device)
            output = clf_FM(batch_train_counts)
            
            loss = loss_fn(output, batch_train_targets)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            
            batch_train_predicts = torch.max(output, dim=1)[1]
            acc = accuracy_score(batch_train_targets.cpu(), batch_train_predicts.cpu())
            
            fm_loss += loss
            fm_acc += acc
            
        fm_loss = fm_loss/int(len(train_index)/batch_size)
        fm_acc = fm_acc/int(len(train_index)/batch_size)
        fm_train_losses.append(fm_loss)
        fm_train_accs.append(fm_acc)
        print('Epoch[{}/{}] Train loss: {:.4f}\tTrain acc: {:.4f}'.format(epoch+1, num_epochs, fm_loss, fm_acc))
        
        clf_FM.eval()
        with torch.no_grad():
            output = clf_FM(torch.FloatTensor(test_counts.toarray()).to(device))
            fm_loss = loss_fn(output, torch.LongTensor(test_targets.map({
                'negative': 0, 'neutral': 1, 'positive': 2
            }).values).to(device))
            test_predicts = torch.max(output, dim=1)[1]
            fm_acc = accuracy_score(torch.LongTensor(test_targets.map({
                'negative': 0, 'neutral': 1, 'positive': 2
            }).values).cpu(), test_predicts.cpu())
            
            fm_test_losses.append(fm_loss)
            fm_test_accs.append(fm_acc)
            print('           Test loss: {:.4f}\tTest acc: {:.4f}'.format(fm_loss, fm_acc))
output = pd.DataFrame(data={'id':range(1, len(test_predicts.cpu())+1), 'polarity': test_predicts.cpu()})
output['polarity'] = output['polarity'].map({0: 'negative', 1: 'neutral', 2: 'positive'})
output.to_csv("./result/fm-not-apply-tfidf.csv", index=False)
def show_curve(ys, title):
    x = np.array(range(len(ys)))
    y = np.array(ys)
    plt.plot(x, y, c='b')
    plt.axis()
    
    plt.title('{} Curve:'.format(title))
    plt.xlabel('Epoch')
    plt.ylabel('{} Value'.format(title))
    plt.show()
    
show_curve(fm_train_losses, 'TRAIN LOSS')
show_curve(fm_test_losses, 'TEST LOSS')
show_curve(fm_train_accs, 'TRAIN ACC')
show_curve(fm_test_accs, 'TEST ACC')<jupyter_output><empty_output><jupyter_text>准确率在 0.80 ~ 0.81 左右<jupyter_code>num_epochs = 6
batch_size = 64
lr = 5e-3
weight_decay = 1e-5
k = 100

device = torch.device("cuda:1")

clf_FM = FM(train_tfidfs.shape[1], k)
clf_FM.to(device)
optimizer = torch.optim.Adam(clf_FM.parameters(), lr=lr, weight_decay=weight_decay)
loss_fn = nn.CrossEntropyLoss()
kf = KFold(n_splits=5, shuffle=True, random_state=0)

fm_train_losses = []
fm_train_accs = []
fm_test_losses = []
fm_test_accs = []

for f, (train_index, val_index) in enumerate(kf.split(train_tfidfs)):
    print('Fold[{}]'.format(f+1))
    for epoch in range(num_epochs):
        clf_FM.train()
        fm_loss = 0
        fm_acc = 0
        for i in range(int(len(train_index)/batch_size)):
            batch_train_tfidfs = Variable(torch.FloatTensor(
                train_tfidfs[train_index][i*batch_size:i*batch_size+batch_size].toarray())).to(device)
            batch_train_targets = Variable(torch.LongTensor(
                train_targets[train_index][i*batch_size:i*batch_size+batch_size].map({
                    'negative': 0, 'neutral': 1, 'positive': 2
                }).values)).to(device)
            output = clf_FM(batch_train_tfidfs)
            
            loss = loss_fn(output, batch_train_targets)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            
            batch_train_predicts = torch.max(output, dim=1)[1]
            acc = accuracy_score(batch_train_targets.cpu(), batch_train_predicts.cpu())
            
            fm_loss += loss
            fm_acc += acc
            
        fm_loss = fm_loss/int(len(train_index)/batch_size)
        fm_acc = fm_acc/int(len(train_index)/batch_size)
        fm_train_losses.append(fm_loss)
        fm_train_accs.append(fm_acc)
        print('Epoch[{}/{}] Train loss: {:.4f}\tTrain acc: {:.4f}'.format(epoch+1, num_epochs, fm_loss, fm_acc))
        
        clf_FM.eval()
        with torch.no_grad():
            output = clf_FM(torch.FloatTensor(test_tfidfs.toarray()).to(device))
            fm_loss = loss_fn(output, torch.LongTensor(test_targets.map({
                'negative': 0, 'neutral': 1, 'positive': 2
            }).values).to(device))
            test_predicts = torch.max(output, dim=1)[1]
            fm_acc = accuracy_score(torch.LongTensor(test_targets.map({
                'negative': 0, 'neutral': 1, 'positive': 2
            }).values).cpu(), test_predicts.cpu())
            
            fm_test_losses.append(fm_loss)
            fm_test_accs.append(fm_acc)
            print('           Test loss: {:.4f}\tTest acc: {:.4f}'.format(fm_loss, fm_acc))
output = pd.DataFrame(data={'id':range(1, len(test_predicts.cpu())+1), 'polarity': test_predicts.cpu()})
output['polarity'] = output['polarity'].map({0: 'negative', 1: 'neutral', 2: 'positive'})
output.to_csv("./result/fm-apply-tfidf.csv", index=False)
def show_curve(ys, title):
    x = np.array(range(len(ys)))
    y = np.array(ys)
    plt.plot(x, y, c='b')
    plt.axis()
    
    plt.title('{} Curve:'.format(title))
    plt.xlabel('Epoch')
    plt.ylabel('{} Value'.format(title))
    plt.show()
    
show_curve(fm_train_losses, 'TRAIN LOSS')
show_curve(fm_test_losses, 'TEST LOSS')
show_curve(fm_train_accs, 'TRAIN ACC')
show_curve(fm_test_accs, 'TEST ACC')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12798766383962992, 'fraction_numerical': 0.039257260344384474, 'mean_word_length': 3.686841312857573, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '318675', 'license': 'MIT', 'n_tokens_mistral': 6278, 'n_tokens_neox': 5798, 'n_words': 967, 'path': 'code/naiveBayes.ipynb', 'repo_name': '2019-sysu-data-mining-final-project/Report', 'size': '15564'}