<jupyter_start><jupyter_text>Verify GPU<jupyter_code>!nvidia-smi<jupyter_output>Sat Apr 24 18:38:27 2021       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |
| N/A   44C    P8    10W /  70W |      0MiB / 15109MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-------[...]<jupyter_text>Weight and Bias (Assisting Metrics, Optional)<jupyter_code>!pip install wandb
!wandb login
project_name = "CoLA with BERT"  # @param {type:"string"}
import os

os.environ["WANDB_PROJECT"] = project_name
import wandb<jupyter_output>Collecting wandb
[?25l  Downloading https://files.pythonhosted.org/packages/f6/28/4aefc543967839bdb4e139831b82004279f1c435cede2a9557ccf8369875/wandb-0.10.27-py2.py3-none-any.whl (2.1MB)
[K     |████████████████████████████████| 2.1MB 8.5MB/s 
[?25hRequirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.1)
Collecting docker-pycreds>=0.4.0
  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl
Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)
Collecting sentry-sdk>=0.4.0
[?25l  Downloading https://files.pythonhosted.org/packages/f3/92/5a33be64990ba815364a8f2dd9e6f51de60d23dfddafb4f1fc5577d4dc64/sentry_sdk-1.0.0-py2.py3-none-any.whl (131kB)
[K     |████████████████████████████████| 133kB 46.9MB/s 
[?25hCollecting pathtools
  Downloading https://files.python[...]<jupyter_text>Installation<jupyter_code>!pip install transformers==4.5.1 datasets==1.5.0 accelerate==0.2.1
import os

import torch
from accelerate import Accelerator
from datasets import load_dataset, load_metric
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss
from torch.utils.data import DataLoader
from tqdm.auto import tqdm
from transformers import (
    AdamW,
    BertConfig,
    BertModel,
    BertTokenizerFast,
    default_data_collator,
    get_linear_schedule_with_warmup,
)
per_device_train_batch_size = 32  # @param {type:"slider", min:1, max:64, step:1}
learning_rate = 2e-5  # @param {type:"slider", min:2e-7, max:2e-3, step:2e-7}
num_train_epochs = 3  # @param {type:"slider", min:1, max:10, step:1}
logging_steps = 10  # @param {type:"slider", min:10, max:100, step:5}<jupyter_output><empty_output><jupyter_text>Tokenizer<jupyter_code>tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased")<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>datasets = load_dataset("glue", "cola")
label_list = datasets["train"].features["label"].names
def preprocess_function(examples):
    # Tokenize the texts
    result = tokenizer(
        examples["sentence"],
        padding="max_length",
        max_length=128,
        truncation="longest_first",
    )
    result["label"] = examples["label"]
    return result
datasets = datasets.map(preprocess_function, batched=True)
datasets["test"] = datasets["test"].remove_columns("label")
for index in range(3):
    print(f"Sample {index} of the training set: {datasets['train'][index]}.")
dataloaders = {
    "train": DataLoader(
        datasets["train"],
        collate_fn=default_data_collator,
        batch_size=per_device_train_batch_size,
        shuffle=True,
    ),
    "validation": DataLoader(
        datasets["validation"],
        collate_fn=default_data_collator,
        batch_size=per_device_train_batch_size,
        shuffle=True,
    ),
    "test": DataLoader(
        datasets["test"],
        collate_fn=default_data_collator,
        batch_size=per_device_train_batch_size,
        shuffle=True,
    ),
}<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class BertClassificationForCoLA(nn.Module):
    def __init__(self, config, from_pretrained=False):
        super().__init__()
        self.config = config
        self.num_labels = config.num_labels

        if not from_pretrained:
            self.bert = BertModel(config)

        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        self.apply(self._init_weights)  # Equivelent of self.init_weights()

    def _init_weights(self, module):
        """ Initialize the weights """
        if isinstance(module, nn.Linear):
            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
            if module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()
        elif isinstance(module, nn.LayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)

    @classmethod
    def from_pretrained(cls, model_name: str, config):
        model = cls(config, from_pretrained=True)
        model.bert = BertModel.from_pretrained(model_name, config=config)
        return model

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        token_type_ids=None,
        position_ids=None,
        head_mask=None,
        inputs_embeds=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        return_dict = (
            return_dict if return_dict is not None else self.config.use_return_dict
        )

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        pooled_output = self.dropout(outputs[1])
        logits = self.classifier(pooled_output)

        loss = None
        if labels is not None:
            if self.num_labels == 1:
                #  We are doing regression
                loss_fct = MSELoss()
                loss = loss_fct(logits.view(-1), labels.view(-1))
            else:
                loss_fct = CrossEntropyLoss()
                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))

        # return final
        return {
            "loss": loss,
            "logits": logits,
        }
config = BertConfig.from_pretrained(
    "bert-base-cased", num_labels=len(label_list), finetuning_task="cola"
)
model = BertClassificationForCoLA.from_pretrained("bert-base-cased", config)<jupyter_output><empty_output><jupyter_text>Optimizer, scheduler, accelerator<jupyter_code>no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
    {
        "params": [
            p
            for n, p in model.named_parameters()
            if not any(nd in n for nd in no_decay)
        ],
        "weight_decay": 0.01,
    },
    {
        "params": [
            p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)
        ],
        "weight_decay": 0.0,
    },
]
optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
num_warmup_steps = 0
scheduler = get_linear_schedule_with_warmup(
    optimizer, num_warmup_steps, num_train_epochs * len(dataloaders["train"])
)
accelerator = Accelerator()
(
    model,
    optimizer,
    dataloaders["train"],
    dataloaders["validation"],
    dataloaders["test"],
) = accelerator.prepare(
    model,
    optimizer,
    dataloaders["train"],
    dataloaders["validation"],
    dataloaders["test"],
)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>run_name = "PyTorch Scratch"
if os.getenv("WANDB_PROJECT"):
    wandb.init(project=project_name, name=run_name)
    wandb.watch(model)
for epoch in range(num_train_epochs):
    model.train()
    for step, batch in enumerate(tqdm(dataloaders["train"])):
        batch.pop("idx")

        optimizer.zero_grad()
        outputs = model(**batch)
        loss = outputs["loss"]
        accelerator.backward(loss)
        optimizer.step()
        scheduler.step()

        if step % logging_steps == 0:
            wandb.log(
                {
                    "train/loss": loss,
                    "train/epoch": epoch + step / len(dataloaders["train"]),
                    "train/learning_rate": scheduler.get_last_lr()[0],
                }
            )
    model.eval()
    metric = load_metric("glue", "cola")
    for batch in tqdm(dataloaders["validation"]):
        batch.pop("idx")

        outputs = model(**batch)
        outputs = accelerator.gather(outputs)
        predictions = outputs["logits"].argmax(dim=-1)
        references = accelerator.gather(batch["labels"])

        metric.add_batch(
            predictions=predictions,
            references=references,
        )

    eval_metric = metric.compute()
    wandb.log({"eval/matthews_correlation": eval_metric["matthews_correlation"]})

os.mkdir(run_name)
accelerator.wait_for_everyone()
model = accelerator.unwrap_model(model)
accelerator.save(model.state_dict(), os.path.join(run_name, "trained_model.pt"))<jupyter_output>[34m[1mwandb[0m: Currently logged in as: [33mtomy0000000[0m (use `wandb login --relogin` to force relogin)<jupyter_text>Test  On dataset<jupyter_code>predictions = torch.empty(0, dtype=torch.int64, device=accelerator.device)
for batch in tqdm(dataloaders["test"]):
    batch.pop("idx")
    outputs = model(**batch)
    predictions = torch.cat((predictions, outputs["logits"].argmax(dim=-1)), 0)
for index, (sample, pred) in enumerate(zip(datasets["test"]["sentence"], predictions)):
    print(f"{index}\t{label_list[pred.item()]}\t{sample}")<jupyter_output>0	acceptable	Bill whistled past the house.
1	acceptable	The car honked its way down the road.
2	acceptable	Bill pushed Harry off the sofa.
3	acceptable	the kittens yawned awake and played.
4	acceptable	I demand that the more John eats, the more he pay.
5	acceptable	If John eats more, keep your mouth shut tighter, OK?
6	acceptable	His expectations are always lower than mine are.
7	acceptable	The sooner you call, the more carefully I will word the letter.
8	unacceptable	The more timid he feels, the more people he interviews without asking questions of.
9	unacceptable	Once Janet left, Fred became a lot crazier.
10	acceptable	If you give every senator enough opportunity, he will succumb to corruption.
11	acceptable	The more time that any senator spends with lobbyists, the more likely he is to succumb to corruption.
12	acceptable	Which problem does the sooner you solve, the more easily you'll satisfy the folks up at corporate headquarters?
13	acceptable	Which problem do you think that the s[...]<jupyter_text>Manually<jupyter_code>sentence = "The probable hostile German reaction is unfortunate."  # @param {type:"string"}
tokenized_input = tokenizer(sentence, return_tensors="pt").to(accelerator.device)
outputs = model(**tokenized_input)
print(f"Prediction: {label_list[outputs['logits'].argmax(dim=-1).item()]}")<jupyter_output>Prediction: acceptable<jupyter_text>Inference<jupyter_code>import torch
from torch import nn
from transformers import BertConfig, BertModel, BertTokenizerFast
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
label_list = ["unacceptable", "acceptable"]
tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased")
config = BertConfig.from_pretrained("bert-base-cased", finetuning_task="cola")
# Scroll to top and run the class definition class first
model = BertClassificationForCoLA.from_pretrained(
    "PyTorch Scratch/trained_model.pt", config
).to(device)
sentence = "The probable hostile German reaction is unfortunate."  # @param {type:"string"}
tokenized_input = tokenizer(sentence, return_tensors="pt").to(device)
outputs = model(**tokenized_input)
print(f"Prediction: {label_list[outputs['logits'].argmax(dim=-1).item()]}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13980839629254616, 'fraction_numerical': 0.026170262481501675, 'mean_word_length': 3.0187793427230045, 'pattern_counts': {'":': 13, '<': 41, '<?xml version=': 0, '>': 44, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '289186', 'license': 'MIT', 'n_tokens_mistral': 4364, 'n_tokens_neox': 4098, 'n_words': 807, 'path': '4 - CoLA Sentences Classification with BERT/Pytorch Scratch.ipynb', 'repo_name': 'NCHU-NLP-Lab/Gym', 'size': '12839'}