<jupyter_start><jupyter_text>Mount Drive<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>UV News Data Input<jupyter_code>import pandas as pd
import numpy as np

true_path = '/content/drive/MyDrive/AI_Final_Project/News_dataset/True.csv'
fake_path = '/content/drive/MyDrive/AI_Final_Project/News_dataset/Fake.csv'

true_df = pd.read_csv(true_path)
fake_df = pd.read_csv(fake_path)
display(true_df)
display(fake_df)<jupyter_output><empty_output><jupyter_text>Data Visulization<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt

sns.countplot(y="subject", palette="coolwarm", data=true_df).set_title('True News Subject Distribution')
plt.show()

sns.countplot(y="subject", palette="coolwarm", data=fake_df).set_title('Fake News Subject Distribution')
plt.show()<jupyter_output><empty_output><jupyter_text>Real News Word Cloud<jupyter_code>from wordcloud import WordCloud

real_titles = true_df.title
real_titles_ls = [text for text in real_titles]
# print(alls)
real_all_words = ' '.join(real_titles)
wordcloud_real = WordCloud(background_color='white',
    width= 800, height= 500,
    max_font_size = 180,
    collocations = False).generate(real_all_words)

plt.figure(figsize=(10,7))
plt.imshow(wordcloud_real, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Fake News Word Cloud<jupyter_code>fake_titles = fake_df.title
fake_titles_ls = [text for text in fake_titles]
# print(alls)
fake_all_words = ' '.join(fake_titles)
wordcloud_fake = WordCloud(background_color='white',
    width= 800, height= 500,
    max_font_size = 180,
    collocations = False).generate(fake_all_words)

plt.figure(figsize=(10,7))
plt.imshow(wordcloud_fake, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Data Preprocessing  Data Combination<jupyter_code># Add Labels to both df
true_df['true'] = 1
fake_df['true'] = 0

# Concat
df = pd.concat([true_df, fake_df])
display(df)<jupyter_output><empty_output><jupyter_text>Inspect Lengths of News<jupyter_code>titles = [text for text in df.title]

max_len = 0
titles_len = []
for title in titles:
    titles_len.append(len(title.split()))
    max_len = max(len(title.split()), max_len)

print('Number of titles:', len(titles))
print('Max length of the titles:', max_len)
print('Mean length of the titles:', np.mean(titles_len))
plt.figure(figsize=(20,5))
g = sns.countplot(x=titles_len)
g.set_xticklabels(g.get_xticklabels(), rotation=50)
plt.show()
texts = [text for text in df.text]

max_len = 0
texts_len = []
for text in texts:
    texts_len.append(len(text.split()))
    max_len = max(len(text.split()), max_len)

# g = sns.countplot(x=texts_len)
print('Mean length of the texts:', np.mean(texts_len))<jupyter_output>Mean length of the texts: 405.28228428883244<jupyter_text>Purify & Shffle the DataFrame<jupyter_code>from sklearn.utils import shuffle

# Purify
df = df.iloc[:,[0, -1]]

# Shuffle
df = shuffle(df).reset_index(drop=True)

display(df)<jupyter_output><empty_output><jupyter_text>Split Data into Training, Validation, Test<jupyter_code>train_val_df = df.sample(frac = 0.8)
test_df = df.drop(train_val_df.index)

train_df = train_val_df.sample(frac = 0.8)
val_df = train_val_df.drop(train_df.index)

# Reset Index
train_df = train_df.reset_index(drop=True)
val_df = val_df.reset_index(drop=True)
test_df = test_df.reset_index(drop=True)

print('trainset size:', train_df.shape)
print('valset size:', val_df.shape)
print('testset size:', test_df.shape)<jupyter_output>trainset size: (28734, 2)
valset size: (7184, 2)
testset size: (8980, 2)<jupyter_text>Data distribution imbalanced<jupyter_code># train_imbalanced = 1/10
# val_imbalanced = 1/10
# test_imbalanced = 1/10

# train_true = train_df[train_df['true'] == 1]
# train_fake = train_df[train_df['true'] == 0]
# train_fake = train_fake.iloc[:round(len(train_fake)*train_imbalanced),:]
# train_true_len = len(train_true)
# train_fake_len = len(train_fake)
# train_df = pd.concat([train_true, train_fake])

# val_true = val_df[val_df['true'] == 1]
# val_fake = val_df[val_df['true'] == 0]
# val_fake = val_fake.iloc[:round(len(val_fake)*val_imbalanced),:]
# val_true_len = len(val_true)
# val_fake_len = len(val_fake)
# val_df = pd.concat([val_true, val_fake])

# test_true = test_df[test_df['true'] == 1]
# test_fake = test_df[test_df['true'] == 0]
# test_fake = test_fake.iloc[:round(len(test_fake)*test_imbalanced),:]
# test_true_len = len(test_true)
# test_fake_len = len(test_fake)
# test_df = pd.concat([test_true, test_fake])

# len_df = pd.DataFrame({'ind': ['Train', 'Validation', 'Test'], 'True': [train_true_len, val_true_len, test_true_len], 'Fake': [train_fake_len, val_fake_len, test_fake_len]})
# len_df.set_index('ind', inplace=True)
# len_df.index.name = None

# len_df<jupyter_output><empty_output><jupyter_text>Dataframe to csv<jupyter_code>train_df.to_csv('train.tsv', sep='\t', index=False)
val_df.to_csv('val.tsv', sep='\t', index=False)
test_df.to_csv('test.tsv', sep='\t', index=False)<jupyter_output><empty_output><jupyter_text>Concatenate all dataframe<jupyter_code>df = pd.concat([train_df, val_df, test_df])
df<jupyter_output><empty_output><jupyter_text>Performing Data Cleaning<jupyter_code>import nltk
# Downloading Stopwords
nltk.download("stopwords")
# Obtaining Additional Stopwords From nltk
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
stop_words.extend(['from', 'subject', 're', 'edu', 'use'])
# Removing Stopwords And Remove Words With 2 Or Less Characters
def preprocess(text):
    result = []
    for token in gensim.utils.simple_preprocess(text):
        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3 and token not in stop_words:
            result.append(token)
            
    return result
import gensim

# Applying The Function To The Dataframe
df['clean'] = df['title'].apply(preprocess)<jupyter_output><empty_output><jupyter_text>Obtaining The Total Words Present In The Dataset<jupyter_code>list_of_words = []
for i in df.clean:
    for j in i:
        list_of_words.append(j)

total_words = len(list(set(list_of_words)))
total_words<jupyter_output><empty_output><jupyter_text>Preparing The Data By Performing Tokenization And Padding<jupyter_code>from nltk import word_tokenize
from tensorflow.keras.preprocessing.text import one_hot, Tokenizer

# Creating A Tokenizer To Tokenize The Words And Create Sequences Of Tokenized Words
tokenizer = Tokenizer(num_words = total_words)
tokenizer.fit_on_texts(train_df['title'])

train_sequences = tokenizer.texts_to_sequences(train_df['title'])
val_sequences = tokenizer.texts_to_sequences(val_df['title'])
test_sequences = tokenizer.texts_to_sequences(test_df['title'])
from tensorflow.keras.preprocessing.sequence import pad_sequences

# Adding Padding
padded_train = pad_sequences(train_sequences,maxlen = 42, padding = 'post', truncating = 'post')
padded_val = pad_sequences(val_sequences,maxlen = 42, padding = 'post', truncating = 'post')
padded_test = pad_sequences(test_sequences,maxlen = 42, padding = 'post', truncating = 'post')<jupyter_output><empty_output><jupyter_text>LSTM  Building And Training LSTM Model<jupyter_code>from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Embedding, Input, LSTM, Conv1D, MaxPool1D, Bidirectional, Dropout, BatchNormalization

## Creating model Using LSTM
embedding_vector_features=40
model=Sequential()
model.add(Embedding(total_words,embedding_vector_features,input_length=20))
model.add(Dropout(0.3))
model.add(LSTM(100))
model.add(Dropout(0.3))
model.add(Dense(1,activation='sigmoid'))
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
print(model.summary())
y_train = np.asarray(train_df['true'])
y_val = np.asarray(val_df['true'])

# Training the model
model.fit(padded_train, y_train, batch_size = 64, validation_data=(padded_val, y_val), epochs = 3)<jupyter_output>Epoch 1/3
WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_input'), name='embedding_input', description="created by layer 'embedding_input'"), but it was called on an input with incompatible shape (None, 42).
WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_input'), name='embedding_input', description="created by layer 'embedding_input'"), but it was called on an input with incompatible shape (None, 42).
447/449 [============================>.] - ETA: 0s - loss: 0.2281 - accuracy: 0.8880WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_input'), name='embedding_input', description="created by layer 'embedding_input'"), but it was called on an input with incompatible [...]<jupyter_text>Assessing Trained Model Performance<jupyter_code># Making prediction
prediction = model.predict_classes(padded_test)

# Getting The Accuracy
from sklearn.metrics import accuracy_score
y_test = np.asarray(test_df['true'])

accuracy = accuracy_score(list(y_test), prediction)

print("LSTM Model Accuracy : ", accuracy)


# Getting The Confusion Matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(list(y_test), prediction)
plt.figure(figsize = (6, 6))
sns.heatmap(cm, annot = True)

from sklearn.metrics import classification_report
print(classification_report(y_test, prediction))<jupyter_output>WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_input'), name='embedding_input', description="created by layer 'embedding_input'"), but it was called on an input with incompatible shape (None, 42).<jupyter_text>Bidirectional LSTM  Building And Training Bidirectional LSTM Model<jupyter_code>## Creating model Using LSTM
embedding_vector_features=40
model=Sequential()
model.add(Embedding(total_words,embedding_vector_features,input_length=20))
model.add(Dropout(0.3))
model.add(Bidirectional(LSTM(100)))
model.add(Dropout(0.3))
model.add(Dense(1,activation='sigmoid'))
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
print(model.summary())
y_train = np.asarray(train_df['true'])
y_val = np.asarray(val_df['true'])

# Training the model
model.fit(padded_train, y_train, batch_size = 64, validation_data=(padded_val, y_val), epochs = 3)<jupyter_output>Epoch 1/3
WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_1_input'), name='embedding_1_input', description="created by layer 'embedding_1_input'"), but it was called on an input with incompatible shape (None, 42).
WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_1_input'), name='embedding_1_input', description="created by layer 'embedding_1_input'"), but it was called on an input with incompatible shape (None, 42).
446/449 [============================>.] - ETA: 0s - loss: 0.1885 - accuracy: 0.9197WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_1_input'), name='embedding_1_input', description="created by layer 'embedding_1_input'"), but it was called on an input [...]<jupyter_text>Assessing Trained Model Performance<jupyter_code># Making prediction
prediction = model.predict_classes(padded_test)

# Getting The Accuracy
y_test = np.asarray(test_df['true'])

accuracy = accuracy_score(list(y_test), prediction)

print("Bidirectional LSTM Model Accuracy : ", accuracy)


# Getting The Confusion Matrix
cm = confusion_matrix(list(y_test), prediction)
plt.figure(figsize = (6, 6))
sns.heatmap(cm, annot = True)

print(classification_report(y_test, prediction))<jupyter_output>WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_1_input'), name='embedding_1_input', description="created by layer 'embedding_1_input'"), but it was called on an input with incompatible shape (None, 42).<jupyter_text>CNN-BiLSTM  Building And Training CNN Model<jupyter_code>## Creating model Using CNN
embedding_vector_features=40
model=Sequential()
model.add(Embedding(total_words,embedding_vector_features,input_length=20))

model.add(Dropout(0.3))

model.add(Conv1D(32, 5, activation='relu'))
model.add(MaxPool1D())

model.add(Conv1D(32, 5, activation='relu'))
model.add(MaxPool1D())

model.add(Bidirectional(LSTM(100)))
model.add(Dropout(0.3))

model.add(Dense(1,activation='sigmoid'))
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
print(model.summary())
y_train = np.asarray(train_df['true'])
y_val = np.asarray(val_df['true'])

# Training the model
model.fit(padded_train, y_train, batch_size = 64, validation_data=(padded_val, y_val), epochs = 3)<jupyter_output>Epoch 1/3
WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_2_input'), name='embedding_2_input', description="created by layer 'embedding_2_input'"), but it was called on an input with incompatible shape (None, 42).
WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_2_input'), name='embedding_2_input', description="created by layer 'embedding_2_input'"), but it was called on an input with incompatible shape (None, 42).
449/449 [==============================] - ETA: 0s - loss: 0.1828 - accuracy: 0.9165WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_2_input'), name='embedding_2_input', description="created by layer 'embedding_2_input'"), but it was called on an input [...]<jupyter_text>Assessing Trained Model Performance<jupyter_code># Making prediction
prediction = model.predict_classes(padded_test)

# Getting The Accuracy
y_test = np.asarray(test_df['true'])

accuracy = accuracy_score(list(y_test), prediction)

print("CNN Model Accuracy : ", accuracy)


# Getting The Confusion Matrix
cm = confusion_matrix(list(y_test), prediction)
plt.figure(figsize = (6, 6))
sns.heatmap(cm, annot = True)

print(classification_report(y_test, prediction))<jupyter_output>WARNING:tensorflow:Model was constructed with shape (None, 20) for input KerasTensor(type_spec=TensorSpec(shape=(None, 20), dtype=tf.float32, name='embedding_2_input'), name='embedding_2_input', description="created by layer 'embedding_2_input'"), but it was called on an input with incompatible shape (None, 42).<jupyter_text>BERT<jupyter_code>!pip install transformers

import torch
from transformers import BertTokenizer
PRETRAINED_MODEL_NAME = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)<jupyter_output><empty_output><jupyter_text>Load Dataset Class<jupyter_code>from torch.utils.data import Dataset

class FakeNewsDataset(Dataset):
    def __init__(self, mode, tokenizer):
        assert mode in ['train', 'val', 'test']
        self.mode = mode
        self.df = pd.read_csv(mode + '.tsv', sep='\t').fillna("")
        self.len = len(self.df)
        self.tokenizer = tokenizer  # BERT tokenizer
    
    # 定義回傳一筆訓練 / 測試數據的函式
    def __getitem__(self, idx):
        if self.mode == 'test':
            statement, label = self.df.iloc[idx, :].values
            label_tensor = torch.tensor(label)
        else:
            statement, label = self.df.iloc[idx, :].values
            label_tensor = torch.tensor(label)
            
        # 建立第一個句子的 BERT tokens 並加入分隔符號 [SEP]
        word_pieces = ['[CLS]']
        statement = self.tokenizer.tokenize(statement)
        word_pieces += statement + ['[SEP]']
        len_st = len(word_pieces)
        
#         # 第二個句子的 BERT tokens
#         tokens_b = self.tokenizer.tokenize(text_b)
#         word_pieces += tokens_b + ["[SEP]"]
#         len_b = len(word_pieces) - len_a
        
        # 將整個 token 序列轉換成索引序列
        ids = self.tokenizer.convert_tokens_to_ids(word_pieces)
        tokens_tensor = torch.tensor(ids)
        
        # 將第一句包含 [SEP] 的 token 位置設為 0
        segments_tensor = torch.tensor([0] * len_st, dtype=torch.long)
        
        return (tokens_tensor, segments_tensor, label_tensor)
    
    def __len__(self):
        return self.len
    
    
# Initialize Datasets for Transformation
trainset = FakeNewsDataset('train', tokenizer=tokenizer)
valset = FakeNewsDataset('val', tokenizer=tokenizer)
testset = FakeNewsDataset('test', tokenizer=tokenizer)

print('trainset size:' ,trainset.__len__())
print('valset size:',valset.__len__())
print('testset size: ',testset.__len__())<jupyter_output>trainset size: 28734
valset size: 7184
testset size:  8980<jupyter_text>Sampling and Observing Tensors<jupyter_code># 選擇第一個樣本
sample_idx = 0

# 將原始文本拿出做比較
statement, label = trainset.df.iloc[sample_idx].values

# 利用剛剛建立的 Dataset 取出轉換後的 id tensors
tokens_tensor, segments_tensor, label_tensor = trainset[sample_idx]

# 將 tokens_tensor 還原成文本
tokens = tokenizer.convert_ids_to_tokens(tokens_tensor.tolist())
combined_text = " ".join(tokens)

# 渲染前後差異，毫無反應就是個 print。可以直接看輸出結果
print(f"""
original_statement: 
{statement}

tokens: 
{tokens}

label: {label}

--------------------

tokens_tensor: 
{tokens_tensor}

segments_tensor: 
{segments_tensor}

label_tensor: 
{label_tensor}

""")<jupyter_output>original_statement: 
¿Qué bolá? Obama jokes with Cuban humorist in video

tokens: 
['[CLS]', '¿', 'que', 'bo', '##la', '?', 'obama', 'jokes', 'with', 'cuban', 'humor', '##ist', 'in', 'video', '[SEP]']

label: 1

--------------------

tokens_tensor: 
tensor([  101,  1094, 10861,  8945,  2721,  1029,  8112, 13198,  2007,  9642,
         8562,  2923,  1999,  2678,   102])

segments_tensor: 
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])

label_tensor: 
1<jupyter_text>Reforming the Dataset to Fit the Model<jupyter_code>from torch.utils.data import DataLoader
from torch.nn.utils.rnn import pad_sequence

# 這個函式的輸入 `samples` 是一個 list，裡頭的每個 element 都是
# 剛剛定義的 `FakeNewsDataset` 回傳的一個樣本，每個樣本都包含 3 tensors：
# - tokens_tensor
# - segments_tensor
# - label_tensor
# 它會對前兩個 tensors 作 zero padding，並產生前面說明過的 masks_tensors
def create_mini_batch(samples):
    tokens_tensors = [s[0] for s in samples]
    segments_tensors = [s[1] for s in samples]
    
    # 測試集有 labels
    if samples[0][2] is not None:
        label_ids = torch.stack([s[2] for s in samples])
    else:
        label_ids = None
    
    # Zero Padding
    tokens_tensors = pad_sequence(tokens_tensors, batch_first=True)
    segments_tensors = pad_sequence(segments_tensors, batch_first=True)
    
    # attention masks，將 tokens_tensors 裡頭不為 zero padding
    # 的位置設為 1 讓 BERT 只關注這些位置的 tokens
    masks_tensors = torch.zeros(tokens_tensors.shape, dtype=torch.long)
    masks_tensors = masks_tensors.masked_fill(tokens_tensors != 0, 1)
    
    return tokens_tensors, segments_tensors, masks_tensors, label_ids


# 初始化一個每次回傳 16 個訓練樣本的 DataLoader
# 利用 `collate_fn` 將 list of samples 合併成一個 mini-batch 是關鍵
BATCH_SIZE = 16
trainloader = DataLoader(trainset, batch_size=BATCH_SIZE, collate_fn=create_mini_batch)
valloader = DataLoader(valset, batch_size=BATCH_SIZE, collate_fn=create_mini_batch)
testloader = DataLoader(testset, batch_size=BATCH_SIZE,collate_fn=create_mini_batch)
data = next(iter(trainloader))

tokens_tensors, segments_tensors, masks_tensors, label_ids = data

print(f"""
tokens_tensors.shape   = {tokens_tensors.shape} 
{tokens_tensors}
------------------------
segments_tensors.shape = {segments_tensors.shape}
{segments_tensors}
------------------------
masks_tensors.shape    = {masks_tensors.shape}
{masks_tensors}
------------------------
label_ids.shape        = {label_ids.shape}
{label_ids}
""")<jupyter_output>tokens_tensors.shape   = torch.Size([16, 22]) 
tensor([[  101,  1094, 10861,  8945,  2721,  1029,  8112, 13198,  2007,  9642,
          8562,  2923,  1999,  2678,   102,     0,     0,     0,     0,     0,
             0,     0],
        [  101,  4924, 23439,  2845,  6624,  1999,  1057,  1012,  1055,  1012,
          1040, 12273, 23707,   102,     0,     0,     0,     0,     0,     0,
             0,     0],
        [  101,  3061,  2600, 16615,  5917, 29158,  8398,  2344,  2006,  7734,
         13117,  1024,  4861,   102,     0,     0,     0,     0,     0,     0,
             0,     0],
        [  101,  7207,  5260,  8398,  2011,  1020,  2685,  2044,  3537,  9530,
          7011,  2497,  1024, 26665,  1013, 12997, 17063,  8554,   102,     0,
             0,     0],
        [  101,  4977,  1005,  1055, 11085,  2243,  9299, 23439,  9989,  1997,
          6206, 11817,   102,     0,     0,     0,     0,     0,     0,     0,
             0,     0],
        [  101,  3422,  3653,  2015,  5261[...]<jupyter_text>Model Construction<jupyter_code>from transformers import BertForSequenceClassification
from IPython.display import display, clear_output

PRETRAINED_MODEL_NAME = "bert-base-uncased"
NUM_LABELS = 2

model = BertForSequenceClassification.from_pretrained(
    PRETRAINED_MODEL_NAME, num_labels=NUM_LABELS)

clear_output()

# high-level 顯示此模型裡的 modules
print("""
name             module
-----------------------""")
for name, module in model.named_children():
    if name == "bert":
        for n, _ in module.named_children():
            print(f"{name}:{n}")
    else:
        print("{:16} {}".format(name, module))
model.config<jupyter_output><empty_output><jupyter_text>Fine-Tuning of BERT<jupyter_code># %%time
from sklearn.metrics import accuracy_score
from tqdm.notebook import tqdm

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("device:", device)
model = model.to(device)

model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
NUM_EPOCHS = 3

for epoch in range(NUM_EPOCHS):
    train_loss = 0.0
    train_acc = 0.0

    loop = tqdm(trainloader)
    for batch_idx, data in enumerate(loop):
        tokens_tensors, segments_tensors, masks_tensors, labels = [t.to(device) for t in data]

        # 將參數梯度歸零
        optimizer.zero_grad()
        
        outputs = model(input_ids=tokens_tensors, 
                        token_type_ids=segments_tensors, 
                        attention_mask=masks_tensors, 
                        labels=labels)

        loss = outputs[0]
        loss.backward()
        optimizer.step()

        logits = outputs[1]
        _, pred = torch.max(logits.data, 1)
        train_acc = accuracy_score(pred.cpu().tolist() , labels.cpu().tolist())

        # 紀錄當前 batch loss
        train_loss += loss.item()

        # if batch_idx == len(trainloader)-1:
        #     _, acc = get_predictions(model, trainloader, compute_acc=True)

        loop.set_description(f"Epoch [{epoch+1}/{NUM_EPOCHS}]")
        loop.set_postfix(acc = train_acc, loss = train_loss)<jupyter_output>device: cuda:0<jupyter_text>Save Model<jupyter_code>torch.save(model, './best_model.pth')
print('Model saved!')<jupyter_output>Model saved!<jupyter_text>Load Model<jupyter_code># model = torch.load('./best_model.pth')
# model = model.to(device)<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay

true=[]
predictions=[]
with torch.no_grad():
    model.eval()
    for data in testloader:
        if next(model.parameters()).is_cuda:
            data = [t.to(device) for t in data if t is not None]
            
        tokens_tensors, segments_tensors, masks_tensors = data[:3]
        test_outputs = model(input_ids=tokens_tensors, 
                    token_type_ids=segments_tensors, 
                    attention_mask=masks_tensors)

        logits = test_outputs[0]
        _, pred = torch.max(logits.data, 1)

        labels = data[3]
        true.extend(labels.cpu().tolist())
        predictions.extend(pred.cpu().tolist())


cm = confusion_matrix(true, predictions, labels=[1, 0], normalize='pred')
print(cm)

disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Real', 'Fake'])
disp.plot()

print('Acc: ', accuracy_score(predictions,true))
df = pd.DataFrame({"pred_label": predictions})

df_pred = pd.concat([testset.df.loc[:, ['title']], 
                          testset.df.loc[:, ['true']], 
                          df.loc[:, 'pred_label']], axis=1)
# df_pred.to_csv('bert_1_prec_training_samples.csv', index=False)
df_pred
print(classification_report(df_pred.true, df_pred.pred_label))<jupyter_output>precision    recall  f1-score   support

           0       0.99      0.98      0.99      4673
           1       0.98      0.99      0.99      4307

    accuracy                           0.99      8980
   macro avg       0.99      0.99      0.99      8980
weighted avg       0.99      0.99      0.99      8980<jupyter_text>Insight on Wrong Classification<jupyter_code>wrong_df = df_pred[df_pred.true != df_pred.pred_label]
sns.countplot(y="true", palette="coolwarm", data=wrong_df).set_title('Wrong Classification Result Real/Fake Distribution')
plt.show()
wrong_titles = df_pred[df_pred.true != df_pred.pred_label].title.values
wrong_titles<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13233346318659914, 'fraction_numerical': 0.03447604207245812, 'mean_word_length': 4.868998628257888, 'pattern_counts': {'":': 2, '<': 111, '<?xml version=': 0, '>': 114, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '31', 'dir': 'jupyter-structured-clean-dedup', 'id': '542394', 'license': 'MIT', 'n_tokens_mistral': 9732, 'n_tokens_neox': 9111, 'n_words': 1863, 'path': 'Balanced_Data.ipynb', 'repo_name': 'wutonytt/NYCU-Spring-2021-AI-Final-Project', 'size': '25670'}