<jupyter_start><jupyter_text><jupyter_code>#@title Dependencies and System Info
!pip install -q transformers
!pip install -q nvidia-ml-py3

# Imports
import pandas as pd
import torch
import numpy as np
from timeit import default_timer as timer
from sklearn.metrics import cohen_kappa_score, accuracy_score
from transformers import (BertTokenizer,
                          BertModel,
                          AdamW,
                          get_linear_schedule_with_warmup,
                          BertForSequenceClassification,
                          TrainingArguments,
                          Trainer)
import pynvml
import datetime
import os
import zipfile
import re
from psutil import virtual_memory
from IPython.display import clear_output

%matplotlib inline
import matplotlib.pyplot as plt
from IPython.display import clear_output, display


# System info
pynvml.nvmlInit()
clear_output()

time_limit = datetime.datetime.now() + datetime.timedelta(hours=21)
ram_gb = virtual_memory().total / 1e9

gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(0)
gpu_name = pynvml.nvmlDeviceGetName(gpu_handle)
gpu_memory = pynvml.nvmlDeviceGetMemoryInfo(gpu_handle).total / (2 ** 30)

print(f"CPU cores: {os.cpu_count()}\n"
      f"Total RAM: {ram_gb:.2f} GB\n"
      "-------------------------------\n"
      f"GPU name: {str(gpu_name, 'utf-8')}\n"
      f"GPU memory: {gpu_memory:.2f} GB\n"
      "-------------------------------\n"
      f"Runtime limit: {time_limit.strftime('%d/%m/%Y %H:%M')}")

pynvml.nvmlShutdown()<jupyter_output>CPU cores: 4
Total RAM: 27.39 GB
-------------------------------
GPU name: Tesla P100-PCIE-16GB
GPU memory: 15.90 GB
-------------------------------
Runtime limit: 05/04/2021 11:11<jupyter_text>Downloading the data<jupyter_code>!git clone https://github.com/roneysco/Fake.br-Corpus
DATA_PATH = "./Fake.br-Corpus/size_normalized_texts"<jupyter_output><empty_output><jupyter_text>Loading the data<jupyter_code>def load_txts(path):
    txts = []
    for filename in sorted(os.listdir(path), key=lambda x: int(re.match("[0-9]+", x).group())):
        with open(os.path.join(path, filename)) as f:
            txts.append(f.read())
    return txts


true_txts = load_txts(os.path.join(DATA_PATH, "true"))
fake_txts = load_txts(os.path.join(DATA_PATH, "fake"))
assert(len(true_txts) == len(fake_txts))

data = pd.DataFrame(
    [{"text": t, "label": 0} for t in true_txts] \
         + [{"text": f, "label": 1}  for f in fake_txts]
).sample(frac=1)

pd.set_option('max_colwidth', 200)
display(data)<jupyter_output><empty_output><jupyter_text>Exploring the data<jupyter_code>sample = data.sample(n=1)
print("> TEXT: \n{}".format(sample["text"].values[0]))
print("> LABEL: {}".format(sample["label"].values[0]))
def count_words(texts):
    counts = []
    for text in texts:
        num_words = len(re.findall(r"[\w']+|[.,!?;:/\"]", text))
        counts.append(num_words)
    return np.array(counts)


counts = count_words(data["text"].values)
intervals = list(range(0, 601, 50))

plt.figure(figsize=(8,6))
plt.rcParams.update({'font.size': 14})
plt.hist(counts, rwidth=0.9, edgecolor='black', linewidth=1, bins=intervals)
plt.xticks(intervals)

plt.xlabel("Number of words and punctuation", fontsize=14, fontweight="bold")
plt.ylabel("Number of texts", fontsize=14, fontweight="bold");<jupyter_output><empty_output><jupyter_text>Splitting the data<jupyter_code>DATA_SPLIT_PC = [0.75, 0.125, 0.125]
assert sum(DATA_SPLIT_PC) == 1

train_df = data.sample(frac=DATA_SPLIT_PC[0])
val_df = data.drop(train_df.index).sample(n=int(len(data) * DATA_SPLIT_PC[1]))
test_df = data.drop(train_df.index).drop(val_df.index)

print("> Training samples: %d" % len(train_df))
print("> Validation samples: %d" % len(val_df))
print("> Test samples: %d" % len(test_df))<jupyter_output>> Training samples: 5400
> Validation samples: 900
> Test samples: 900<jupyter_text>Making the datasets compatible with BERT<jupyter_code># BERT tokenizer
bert_tokenizer = BertTokenizer.from_pretrained("neuralmind/bert-base-portuguese-cased",
                                               do_lower_case=False)
class BertFakeNewsDataset(torch.utils.data.Dataset):
    def __init__(self, df, tokenizer=bert_tokenizer, max_seq_length=512):
        self._labels = torch.tensor(df["label"].values, dtype=torch.long)
        self._encodings = {"input_ids": [],
                           "token_type_ids": [],
                           "attention_mask": []}

        for txt in df["text"].values:
            enc_dict = tokenizer.encode_plus(
                text=txt,
                add_special_tokens=True,
                max_length=max_seq_length,
                return_token_type_ids=True,
                padding="max_length",
                return_attention_mask=True,
                return_tensors="pt",
                truncation=True,
            )
            for k, v in enc_dict.items():
                self._encodings[k].append(v[0])
    
    def __getitem__(self, idx):
        item = {key: value[idx] for key, value in self._encodings.items()}
        item["labels"] = self._labels[idx]
        return item

    def __len__(self):
        return len(self._labels)
train_ds = BertFakeNewsDataset(train_df)
val_ds = BertFakeNewsDataset(val_df)
test_ds = BertFakeNewsDataset(test_df)
train_ds[0]<jupyter_output><empty_output><jupyter_text>Loading a BERT model<jupyter_code>model = BertForSequenceClassification.from_pretrained(
    "neuralmind/bert-base-portuguese-cased",
    num_labels=2,
)
sample = {k: torch.unsqueeze(v, 0)
          for k, v in train_ds[0].items()}

start_time = timer()
print(f"Output: {model(**sample)}")
print(f"Time: {timer() - start_time:.5f}s")<jupyter_output>Output: SequenceClassifierOutput(loss=tensor(0.6572, grad_fn=<NllLossBackward>), logits=tensor([[-0.0115,  0.0618]], grad_fn=<AddmmBackward>), hidden_states=None, attentions=None)
Time: 0.76128s<jupyter_text>Training the model<jupyter_code>def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    return {
        "accuracy": accuracy_score(labels, predictions),
        "qwk": cohen_kappa_score(labels, predictions, weights="quadratic"),
    }
train_args = TrainingArguments(
    output_dir="drive/MyDrive/Colab Notebooks/saved_models/fake_news_bertimbau",
    overwrite_output_dir=True,
    evaluation_strategy="epoch",
    learning_rate=2.5e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=10,
    weight_decay=0.01,
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
    save_strategy="steps",
    save_steps=1000,
    save_total_limit=10,
)

trainer = Trainer(
    model=model,
    args=train_args,
    train_dataset=train_ds,
    eval_dataset=val_ds,
    tokenizer=bert_tokenizer,
    compute_metrics=compute_metrics
)
trainer.train()<jupyter_output><empty_output><jupyter_text>Evaluating the model<jupyter_code>trainer.evaluate(train_ds)
trainer.evaluate(val_ds)
trainer.evaluate(test_ds)<jupyter_output><empty_output><jupyter_text>Saving the best model and generating PDF report<jupyter_code>model.save_pretrained("drive/MyDrive/Colab Notebooks/saved_models/fake_news_bertimbau/best_model")
!apt-get install texlive texlive-xetex texlive-latex-extra pandoc
!pip install pypandoc
!jupyter nbconvert --to PDF "drive/MyDrive/Colab Notebooks/fake_news_bertimbau.ipynb"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.142491926803014, 'fraction_numerical': 0.018702906350914962, 'mean_word_length': 3.7374123645634163, 'pattern_counts': {'":': 9, '<': 40, '<?xml version=': 0, '>': 48, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '661311', 'license': 'MIT', 'n_tokens_mistral': 2699, 'n_tokens_neox': 2577, 'n_words': 446, 'path': 'br_fake_news_bertimbau.ipynb', 'repo_name': 'Talendar/br_fake_news_detection', 'size': '7432'}