<jupyter_start><jupyter_text>Setting Workspace<jupyter_code>!pip install ipython-autotime
!pip install transformers
%load_ext autotime
from google.colab import drive
drive.mount('/content/drive')
!pwd
import os
os.chdir('/content/drive/My Drive/HLT/')
!pwd<jupyter_output>/content
/content/drive/My Drive/HLT
time: 241 ms (started: 2021-08-09 08:50:59 +00:00)<jupyter_text>Import Libraries<jupyter_code>import pandas as pd
import numpy as np
from transformers import AutoTokenizer, pipeline, TFBertModel
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report,roc_auc_score,roc_curve,r2_score,recall_score,confusion_matrix,precision_recall_curve
from sklearn.metrics import accuracy_score
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.naive_bayes import MultinomialNB
from sklearn.naive_bayes import GaussianNB
import numpy as np
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
import matplotlib.pyplot as plt
from tensorflow.keras.layers import InputLayer, Embedding, Concatenate, TimeDistributed, Bidirectional, GRU
from tensorflow.keras.layers import LSTM, Dense,Flatten,Conv2D,Conv1D,GlobalMaxPooling1D,GlobalMaxPool1D
from sklearn.metrics import classification_report,roc_auc_score,roc_curve,r2_score,recall_score,confusion_matrix,precision_recall_curve
import seaborn as sns
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import Dropout
pd.set_option('display.max_colwidth', -1) #show all text in col
pd.set_option('display.max_rows', None) #show all rows<jupyter_output>time: 3.47 ms (started: 2021-08-09 08:51:12 +00:00)<jupyter_text>Create Bert Embeddings<jupyter_code>train_df = pd.read_csv("datasets/train.csv") 
train_df['headline']=train_df['headline'].apply(str)
train_df.head()
test_df = pd.read_csv("datasets/test.csv") 
test_df['headline']=test_df['headline'].apply(str)
test_df.head()
print("Train size:{}".format(train_df.shape))
print("Test size:{}".format(test_df.shape))
train_y = train_df['is_sarcastic']
test_y = test_df['is_sarcastic']

train_x = train_df['headline']
test_x = test_df['headline']
train_x.shape,train_y.shape,test_x.shape,test_y.shape
# BERT Embeddings

def transformer_embedding(name,inp,model_name):

    model = model_name.from_pretrained(name)
    tokenizer = AutoTokenizer.from_pretrained(name)
    pipe = pipeline('feature-extraction', model=model, 
                tokenizer=tokenizer)
    features = pipe(inp)
    features = np.squeeze(features)
    return features
'''
Embedding trainset

'''
train_x_list = train_x.tolist()
train_y_list = train_y.to_list()
len(train_x_list)
len(train_y_list)<jupyter_output><empty_output><jupyter_text>Train 1<jupyter_code># Embeddings trainset
embedding_features=transformer_embedding('bert-base-uncased',train_x_list[0:4000],TFBertModel)
embedding_features.shape
train_bert_x = embedding_features[:,0,:]
train_bert_x.shape
train_bert_x_df = pd.DataFrame(train_bert_x)
train_bert_x_df.head()
train_bert_x_df['is_sarcastic'] = train_y_list[0:4000]
train_bert_x_df.head()
train_bert_x_df.to_csv('datasets/BERT/train_bert_0_4000.csv', encoding='utf-8', index=False)<jupyter_output><empty_output><jupyter_text>Train 2<jupyter_code># Embeddings trainset
embedding_features=transformer_embedding('bert-base-uncased',train_x_list[4000:8000],TFBertModel)
embedding_features.shape
train_bert_x = embedding_features[:,0,:]
train_bert_x.shape
train_bert_x_df = pd.DataFrame(train_bert_x)
train_bert_x_df.head()
train_bert_x_df['is_sarcastic'] = train_y_list[4000:8000]
train_bert_x_df.head()
train_bert_x_df.to_csv('datasets/BERT/train_bert_4000_8000.csv', encoding='utf-8', index=False)<jupyter_output><empty_output><jupyter_text>Train 3<jupyter_code># Embeddings trainset
embedding_features=transformer_embedding('bert-base-uncased',train_x_list[8000:12000],TFBertModel)
embedding_features.shape
train_bert_x = embedding_features[:,0,:]
train_bert_x.shape
train_bert_x_df = pd.DataFrame(train_bert_x)
train_bert_x_df.head()
train_bert_x_df['is_sarcastic'] = train_y_list[8000:12000]
train_bert_x_df.head()
train_bert_x_df.to_csv('datasets/BERT/train_bert_8000_12000.csv', encoding='utf-8', index=False)<jupyter_output><empty_output><jupyter_text>Train 4<jupyter_code># Embeddings trainset
embedding_features=transformer_embedding('bert-base-uncased',train_x_list[12000:16000],TFBertModel)
embedding_features.shape
train_bert_x = embedding_features[:,0,:]
train_bert_x.shape
train_bert_x_df = pd.DataFrame(train_bert_x)
train_bert_x_df.head()
train_bert_x_df['is_sarcastic'] = train_y_list[12000:16000]
train_bert_x_df.head()
train_bert_x_df.to_csv('datasets/BERT/train_bert_12000_16000.csv', encoding='utf-8', index=False)<jupyter_output><empty_output><jupyter_text>Train 5<jupyter_code># Embeddings trainset
embedding_features=transformer_embedding('bert-base-uncased',train_x_list[16000:20000],TFBertModel)
embedding_features.shape
train_bert_x = embedding_features[:,0,:]
train_bert_x.shape
train_bert_x_df = pd.DataFrame(train_bert_x)
train_bert_x_df.head()
train_bert_x_df['is_sarcastic'] = train_y_list[16000:20000]
train_bert_x_df.head()
train_bert_x_df.to_csv('datasets/BERT/train_bert_16000_20000.csv', encoding='utf-8', index=False)<jupyter_output><empty_output><jupyter_text>Train 6<jupyter_code># Embeddings trainset
embedding_features=transformer_embedding('bert-base-uncased',train_x_list[20000:],TFBertModel)
embedding_features.shape
train_bert_x = embedding_features[:,0,:]
train_bert_x.shape
train_bert_x_df = pd.DataFrame(train_bert_x)
train_bert_x_df.head()
train_bert_x_df['is_sarcastic'] = train_y_list[20000:]
train_bert_x_df.head()
train_bert_x_df.to_csv('datasets/BERT/train_bert_20000_end.csv', encoding='utf-8', index=False)<jupyter_output>time: 3.83 s (started: 2021-08-09 08:26:51 +00:00)<jupyter_text>Test 1<jupyter_code>'''
Embedding testset

'''
test_x_list = test_x.tolist()
test_y_list = test_y.to_list()
len(test_x_list)
len(test_y_list)
# Embeddings trainset
embedding_features=transformer_embedding('bert-base-uncased',test_x_list,TFBertModel)
embedding_features.shape
test_bert_x = embedding_features[:,0,:]
test_bert_x.shape
test_bert_x_df = pd.DataFrame(test_bert_x)
test_bert_x_df.head()
test_bert_x_df['is_sarcastic'] = test_y_list
test_bert_x_df.head()
test_bert_x_df.to_csv('datasets/BERT/test_bert.csv', encoding='utf-8', index=False)<jupyter_output>time: 5.64 s (started: 2021-08-09 08:30:29 +00:00)<jupyter_text>Merge Trainset<jupyter_code>train_bert_1 = pd.read_csv("datasets/BERT/train_bert_0_4000.csv") 
train_bert_2 = pd.read_csv("datasets/BERT/train_bert_4000_8000.csv")
train_bert_3 = pd.read_csv("datasets/BERT/train_bert_8000_12000.csv") 
train_bert_4 = pd.read_csv("datasets/BERT/train_bert_12000_16000.csv")
train_bert_5 = pd.read_csv("datasets/BERT/train_bert_16000_20000.csv")
train_bert_6 = pd.read_csv("datasets/BERT/train_bert_20000_end.csv")    
train_bert_df = train_bert_1.append([train_bert_2, train_bert_3, train_bert_4, train_bert_5, train_bert_6])
train_bert_df.shape
train_bert_df.to_csv('datasets/BERT/train_bert.csv', encoding='utf-8', index=False)<jupyter_output>time: 45.2 s (started: 2021-08-09 08:30:48 +00:00)<jupyter_text>Statistical Models<jupyter_code>train_bert_df = pd.read_csv("datasets/BERT/train_bert.csv") 
train_bert_df.shape 
train_bert_df.head()
test_bert_df = pd.read_csv("datasets/BERT/test_bert.csv") 
test_bert_df.shape 
train_y = train_bert_df.pop('is_sarcastic')
test_y = test_bert_df.pop('is_sarcastic')

train_x = train_bert_df
test_x = test_bert_df

train_x.shape,train_y.shape,test_x.shape,test_y.shape<jupyter_output><empty_output><jupyter_text>SVC<jupyter_code>from sklearn.svm import SVC
svc = SVC(kernel ='rbf', gamma = 0.1, C = 10, probability=True) #2h 56min
svc.fit(train_x,train_y)
pred=svc.predict(test_x)
print("Evaluate confusion matrix for LR")
print(confusion_matrix(test_y,pred))
print(f"Accuracy Score for LR with C=1.0  ={accuracy_score(test_y,pred)}")<jupyter_output>Evaluate confusion matrix for LR
[[1990  258]
 [ 664 1095]]
Accuracy Score for LR with C=1.0  =0.7699026703269278
time: 15.6 ms (started: 2021-08-09 11:49:25 +00:00)<jupyter_text>Logistic Regression<jupyter_code># Applying Logistic Regression on split tfidf baseline
model=LogisticRegression(max_iter=10000)
model.fit(train_x,train_y)
pred=model.predict(test_x)
print("Evaluate confusion matrix for LR")
print(confusion_matrix(test_y,pred))
print(f"Accuracy Score for LR with C=1.0  ={accuracy_score(test_y,pred)}")<jupyter_output>Evaluate confusion matrix for LR
[[1860  388]
 [ 461 1298]]
Accuracy Score for LR with C=1.0  =0.7881207886199152
time: 25.4 s (started: 2021-08-09 11:49:25 +00:00)<jupyter_text>KNN<jupyter_code>knn = KNeighborsClassifier(n_neighbors = 25, weights = 'distance')
knn.fit(train_x,train_y)
pred=knn.predict(test_x)
print("Evaluate confusion matrix for LR")
print(confusion_matrix(test_y,pred))
print(f"Accuracy Score for LR with C=1.0  ={accuracy_score(test_y,pred)}")<jupyter_output>Evaluate confusion matrix for LR
[[1805  443]
 [ 541 1218]]
Accuracy Score for LR with C=1.0  =0.7544297479411031
time: 10.5 ms (started: 2021-08-09 11:52:39 +00:00)<jupyter_text>Decision Tree<jupyter_code>dt = DecisionTreeClassifier(criterion ='entropy', max_depth= None, min_samples_leaf=1, min_samples_split=20,splitter="best")
dt.fit(train_x,train_y)
pred=dt.predict(test_x)
print("Evaluate confusion matrix for LR")
print(confusion_matrix(test_y,pred))
print(f"Accuracy Score for LR with C=1.0  ={accuracy_score(test_y,pred)}")<jupyter_output>Evaluate confusion matrix for LR
[[1534  714]
 [ 771  988]]
Accuracy Score for LR with C=1.0  =0.6293985525330671
time: 9.04 ms (started: 2021-08-09 11:53:28 +00:00)<jupyter_text>Random Forest<jupyter_code>rf = RandomForestClassifier(criterion='entropy',n_estimators = 100, max_depth = None, bootstrap = False)
rf.fit(train_x,train_y)
pred=rf.predict(test_x)
print("Evaluate confusion matrix for LR")
print(confusion_matrix(test_y,pred))
print(f"Accuracy Score for LR with C=1.0  ={accuracy_score(test_y,pred)}")<jupyter_output>Evaluate confusion matrix for LR
[[1925  323]
 [ 720 1039]]
Accuracy Score for LR with C=1.0  =0.7397055153481408
time: 10.5 ms (started: 2021-08-09 11:56:37 +00:00)<jupyter_text>Naive Bayes<jupyter_code>gnb = GaussianNB()
gnb.fit(train_x,train_y)
pred=gnb.predict(test_x)
print("Evaluate confusion matrix for LR")
print(confusion_matrix(test_y,pred))
print(f"Accuracy Score for LR with C=1.0  ={accuracy_score(test_y,pred)}")<jupyter_output>Evaluate confusion matrix for LR
[[1581  667]
 [ 585 1174]]
Accuracy Score for LR with C=1.0  =0.6875467931120539
time: 15.4 ms (started: 2021-08-09 11:56:37 +00:00)
{'fraction_non_alphanumeric': 0.11853408883202339, 'fraction_numerical': 0.06296837872418205, 'mean_word_length': 9.227102803738317, 'pattern_counts': {'":': 0, '<': 62, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '316371', 'license': 'MIT', 'n_tokens_mistral': 4786, 'n_tokens_neox': 4229, 'n_words': 646, 'path': 'BERT.ipynb', 'repo_name': 'luca-demartino/sarcasm-detector', 'size': '10942'}