<jupyter_start><jupyter_text>tf<jupyter_code>vector = TfidfVectorizer(max_features=1000)
train_tfidf = vector.fit_transform(X_train['review'].apply(lambda x: np.str_(x)))
test_tfidf = vector.transform(X_test['review'].apply(lambda x: np.str_(x)))
df_tfidf_train = pd.DataFrame(train_tfidf.todense())
df_tfidf_test = pd.DataFrame(test_tfidf.todense())
print(df_tfidf_train.shape)
print(df_tfidf_test.shape)<jupyter_output>(168002, 1000)
(42001, 1000)<jupyter_text>Enhanced features n-grams<jupyter_code>import nltk
nltk.download('stopwords')
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from nltk.corpus import stopwords
from sklearn.feature_selection import SelectKBest
en_stopwords = set(stopwords.words("english"))
cout = CountVectorizer(ngram_range=(1,3), analyzer='word', lowercase=True, stop_words=en_stopwords)
vec_train_ngram = cout.fit_transform(X_train['review'].apply(lambda x: np.str_(x)))
#vec_train_ngram = cout.transform().toarray()
vec_test_ngram = cout.transform(X_test['review'].apply(lambda x: np.str_(x)))
print(vec_train_ngram.shape)
print(vec_test_ngram.shape)

skb = SelectKBest(k=600).fit(vec_train_ngram, y_train)
train_ngram = skb.transform(vec_train_ngram)
test_ngram = skb.transform(vec_test_ngram)
print(train_ngram.shape)
print(test_ngram.shape)
df_ngram_train = pd.DataFrame(train_ngram.todense())
df_ngram_test = pd.DataFrame(test_ngram.todense())<jupyter_output>(168002, 600)
(42001, 600)<jupyter_text>Lexicon-based feature<jupyter_code>from tqdm import tqdm
from collections import Counter
import nltk
from nltk.tokenize import word_tokenize
import os
import glob
import time
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from nltk.tokenize import TweetTokenizer
from nltk.tokenize import word_tokenize
import re
def preprocess_lexicon(uni_txt, bi_txt, train_set, test_set):
    glob.glob("lexica/*/*")
    d_uni = {}
    for i in [i.strip() for i in open(uni_txt).readlines()]:
        d_uni[i.split("\t")[0]] = i.split("\t")[1:]
    d_bi = {}
    # for i in [i.strip() for i in open(bi_txt).readlines()]:
    #     d_bi[i.split("\t")[0]] = i.split("\t")[1:]

    train_14feature = preprocess_check_data_in_lexicon(train_set, d_uni, d_bi)
    #train_14feature=[[]]
    #print('test_set shape is,',test_set.shape)
    test_14feature = preprocess_check_data_in_lexicon(test_set, d_uni, d_bi)
    print('test_14feature has the shape,', test_14feature.shape)
    return train_14feature, test_14feature
def preprocess_check_data_in_lexicon(dataset, d_uni,d_bi):
    unilist = d_uni.keys()
    un_usekey = {}
    for i in tqdm(list(dataset["review"])):
      wordlist = i.split()
      for word in wordlist:
        if word in unilist:
          un_usekey[word] = d_uni[word]
        # for key in d_uni.keys():
        #     if key in i:
        #         un_usekey[key] = d_uni[key]
        else:
               pass
    bi_usekey = {}
    # for i in tqdm(list(dataset["review"])):
    #     for key in d_bi.keys():
    #         if key in i:
    #             bi_usekey[key] = d_bi[key]
    #         else:
    #             pass
    return get_feature_vec(dataset, un_usekey, bi_usekey)
def get_feature_vec(dataset, un_usekey, bi_usekey):

    un_vec = getother_feature_gram(un_usekey, list(dataset["review"]), sign="un")
    print('unvec shape is ', un_vec.shape)
    # bi_vec = getother_feature_gram(bi_usekey,list(dataset["review"]), sign="bi")
    # vec_lexfeature = np.concatenate([un_vec,bi_vec],axis=1)
    #print('vec_lexfeature shape is ', vec_lexfeature.shape)
    print('vec_lexfeature shape is ', un_vec.shape)
    return un_vec

def getother_feature_gram(d_bi, dfall, sign="bi"):
    pos_token_num = []
    neg_token_num = []
    score_all = []
    pos_max_all = []
    neg_max_all = []
    neg_last_all = []
    pos_last_all = []
    for key in d_bi.keys():
        d_bi[key][0] = float(d_bi[key][0])
    for i in tqdm(dfall):
        pos_token = 0
        neg_token = 0
        score = 0
        pos_max = 0
        neg_max = 0

        for key in d_bi.keys():
            if key in i:
                if d_bi[key][0] >= 0:
                    pos_token += 1
                else:
                    neg_token += 1
                score += d_bi[key][0]
                pos_max = max(pos_max, d_bi[key][0])
                neg_max = min(neg_max, d_bi[key][0])
        pos_token_num.append(pos_token)
        neg_token_num.append(neg_token)
        score_all.append(score)
        pos_max_all.append(pos_max)
        neg_max_all.append(neg_max)
        pos_last = 0
        neg_last = 0
        if sign == "un":
            i = i.split(" ")
            for z in i[::-1]:
                if z in d_bi.keys():
                    if d_bi[z][0] >= 0:
                        pos_last = d_bi[z][0]
                        break
            for z in i[::-1]:
                if z in d_bi.keys():
                    if d_bi[key][0] < 0:
                        neg_last = d_bi[key][0]
                        break
        elif sign == "bi":
            i = i.split(" ")
            for z in reversed(range(len(i))):
                if z - 1 >= 0:
                    key = " ".join(i[z - 1:z + 1])
                    if key in d_bi.keys():
                        if d_bi[key][0] >= 0:
                            pos_last = d_bi[key][0]
                            break
            for z in reversed(range(len(i))):
                if z - 1 >= 0:
                    key = " ".join(i[z - 1:z + 1])
                    if key in d_bi.keys():
                        if d_bi[key][0] < 0:
                            neg_last = d_bi[key][0]
                            break

        neg_last_all.append(neg_last)
        pos_last_all.append(pos_last)
    pos_token_num = np.array(pos_token_num).reshape(-1, 1)
    neg_token_num = np.array(neg_token_num).reshape(-1, 1)
    score_all = np.array(score_all).reshape(-1, 1)
    pos_max_all = np.array(pos_max_all).reshape(-1, 1)
    neg_max_all = np.array(neg_max_all).reshape(-1, 1)
    neg_last_all = np.array(neg_last_all).reshape(-1, 1)
    pos_last_all = np.array(pos_last_all).reshape(-1, 1)
    return np.concatenate([pos_token_num, neg_token_num, score_all, pos_max_all, neg_max_all, neg_last_all, pos_last_all], axis=1)
uni_txt = "/content/gdrive/MyDrive/6893/Emoticon-unigrams.txt"
bi_txt = "/content/gdrive/MyDrive/6893/Emoticon-bigrams.txt"
train_set, test_set = X_train, X_test
train_lexicon, test_lexicon = preprocess_lexicon(uni_txt,bi_txt,train_set,test_set)
df_lexicon_train = pd.DataFrame(train_lexicon)
df_lexicon_test = pd.DataFrame(test_lexicon)
print(df_lexicon_train.shape)
print(df_lexicon_test.shape)
df_lexicon_train.to_csv('/content/gdrive/MyDrive/6893/df_lexicon_train.csv')
df_lexicon_test.to_csv('/content/gdrive/MyDrive/6893/df_lexicon_test.csv')
df_lexicon_train= pd.read_csv('/content/gdrive/MyDrive/6893/df_lexicon_train.csv')
df_lexicon_test=pd.read_csv('/content/gdrive/MyDrive/6893/df_lexicon_test.csv')
df_enhanced_train = pd.concat([df_ngram_train,df_lexicon_train],axis=1)
df_enhanced_test = pd.concat([df_ngram_test,df_lexicon_test],axis=1)
df_enhanced_train = df_ngram_train
df_enhanced_test = df_ngram_test
print(df_enhanced_train.shape)
print(df_enhanced_test.shape)<jupyter_output><empty_output><jupyter_text>Model  Naive Bayes<jupyter_code>from sklearn.metrics import classification_report
from sklearn.naive_bayes import GaussianNB
gnb = GaussianNB()
gnb.fit(df_enhanced_train, y_train)
print(classification_report(gnb.predict(df_enhanced_test), y_test,digits=3))
y_pred = gnb.predict(df_enhanced_test)<jupyter_output>precision    recall  f1-score   support

           0      0.906     0.660     0.764     29036
           1      0.526     0.848     0.649     12965

    accuracy                          0.718     42001
   macro avg      0.716     0.754     0.706     42001
weighted avg      0.789     0.718     0.728     42001<jupyter_text>SVM<jupyter_code># from sklearn.svm import SVC
# svc = SVC(C=1, kernel="rbf",gamma='auto')
# svc.fit(df_tfidf_train, y_train)
# print(classification_report(svc.predict(df_tfidf_test), y_test))<jupyter_output><empty_output><jupyter_text>Logestic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()
lr.fit(df_enhanced_train, y_train)
print(classification_report(lr.predict(df_enhanced_test), y_test,digits=3))
y_pred = lr.predict(df_enhanced_test)<jupyter_output><empty_output><jupyter_text>RandomForest<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
clf = RandomForestClassifier(max_depth=7, random_state=0)
clf.fit(df_enhanced_train, y_train)
#df_tfidf_test
print(classification_report(clf.predict(df_enhanced_test), y_test,digits=3))
y_pred = clf.predict(df_enhanced_test)
print(classification_report(clf.predict(df_tfidf_test), y_test))<jupyter_output>precision    recall  f1-score   support

           0       0.84      0.72      0.77     24643
           1       0.67      0.80      0.73     17358

    accuracy                           0.75     42001
   macro avg       0.75      0.76      0.75     42001
weighted avg       0.77      0.75      0.75     42001<jupyter_text>NN<jupyter_code>from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegression
NN = MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(128, 2), random_state=1)
#LR = LogisticRegression(random_state=0).fit(X_train, Y_train)
NN.fit(df_enhanced_train, y_train)
#LR.fit(X_train, Y_train)
print(classification_report(NN.predict(df_enhanced_test), y_test))
y_pred = NN.predict(df_enhanced_test)
acc = 0
y_test=list(y_test)
for i in range(len(y_pred)):
  if y_pred[i]==y_test[i]:
    acc += 1
acc = acc/len(y_pred)
acc
fpr, tpr, thresholds = metrics.roc_curve(list(y_test), list(y_pred), pos_label=1)
metrics.auc(fpr, tpr)
from sklearn.metrics import roc_auc_score
roc_auc_score(y_test, y_pred)
from nltk.corpus import stopwords
list_dimensions = [1000,2000,3000,4000,5000,6000,7000]
for k in list_dimensions:
  en_stopwords = set(stopwords.words("english"))
  vector = TfidfVectorizer(max_features=k,stop_words=en_stopwords)
  train_tfidf = vector.fit_transform(X_train['review'].apply(lambda x: np.str_(x)))
  test_tfidf = vector.transform(X_test['review'].apply(lambda x: np.str_(x)))
  df_tfidf_train = pd.DataFrame(train_tfidf.todense())
  df_tfidf_test = pd.DataFrame(test_tfidf.todense())
  lr = LogisticRegression()
  lr.fit(df_tfidf_train, y_train)
  print(classification_report(lr.predict(df_tfidf_test), y_test,digits=3))
en_stopwords = set(stopwords.words("english"))
vector = TfidfVectorizer(max_features=2000,stop_words=en_stopwords)
train_tfidf = vector.fit_transform(X_train['review'].apply(lambda x: np.str_(x)))
test_tfidf = vector.transform(X_test['review'].apply(lambda x: np.str_(x)))
df_tfidf_train = pd.DataFrame(train_tfidf.todense())
df_tfidf_test = pd.DataFrame(test_tfidf.todense())
lr = LogisticRegression()
lr.fit(df_tfidf_train, y_train)
print(classification_report(lr.predict(df_tfidf_test), y_test,digits=3))
y_pred = lr.predict(df_tfidf_test)
fpr, tpr, thresholds = metrics.roc_curve(list(y_test), list(y_pred), pos_label=1)
metrics.auc(fpr, tpr)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12043601559730592, 'fraction_numerical': 0.03030840127614321, 'mean_word_length': 3.476398254660849, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '327032', 'license': 'MIT', 'n_tokens_mistral': 4461, 'n_tokens_neox': 4191, 'n_words': 680, 'path': 'notebooks/experiment_tf_enhanced.ipynb', 'repo_name': 'Sapphirine/202112-20-Sentiment-Prediction-of-Game-Reviews', 'size': '11284'}