<jupyter_start><jupyter_text>From:- [BERT Fine-Tuning Tutorial with PyTorch · <NAME>](http://mccormickml.com/2019/07/22/BERT-fine-tuning/)- [huggingface/pytorch-transformers: 👾 A library of state-of-the-art pretrained models for Natural Language Processing (NLP)](https://github.com/huggingface/pytorch-transformers)Fine-Tuning:- Easy Training: recommend 2-4 epochs on a special NLP task- Less Data- Good Results<jupyter_code>import torch
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from pytorch_transformers import BertTokenizer, BertConfig
from pytorch_transformers import BertForSequenceClassification, BertModel
from pytorch_transformers.optimization import AdamW, WarmupLinearSchedule
from tqdm import tqdm, trange
import pandas as pd
import io
import os
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Data ```bash download glue data$ git clone https://gist.github.com/60c2bdb54d156a41194446737ce03e2e.git download_glue_repo$ python download_glue_repo/download_glue_data.py --data_dir='glue_data'```<jupyter_code># [The Corpus of Linguistic Acceptability (CoLA)](https://nyu-mll.github.io/CoLA/)
data_path = "cola_public/raw/"
train_path = os.path.join(data_path, "in_domain_train.tsv")
df = pd.read_csv(train_path, delimiter='\t', header=None, 
                 names=['sentence_source', 'label', 'label_notes', 'sentence'])
df.sample(5)
sentences = df.sentence.values
sentences = ["[CLS] " + sentence + " [SEP]" for sentence in sentences]
labels = df.label.values<jupyter_output><empty_output><jupyter_text>Input<jupyter_code># The default download path is ~/.cache
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
tokenized_texts = [tokenizer.tokenize(sent) for sent in sentences]
print ("Tokenize the first sentence:")
print (tokenized_texts[0])
tokenizer.tokenize("i love you")
tokenizer.vocab_size<jupyter_output><empty_output><jupyter_text>Format:- input ids: index in BERT tokenizer vocabulary- segment mask: a sequence of 1s and 0s used to identify whether the input is one sentence or two sentences.- attention mask: a sequence of 1s and 0s, with 1s for input tokens and 0s for padding ones.- labels: a single 0 or 1<jupyter_code>MAX_LEN = 16
input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(s) for s in tokenized_texts],
                         maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")
# "post" means padding or truncating at the end of the sequence.
attn_masks = []
for seq in input_ids:
    seq_mask = [float(i>0) for i in seq]
    attn_masks.append(seq_mask)
len(attn_masks)
len(input_ids)
train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(
    input_ids, labels, random_state=2018, test_size=0.1)
train_masks, validation_masks, _, _ = train_test_split(
    attn_masks, input_ids, random_state=2018, test_size=0.1)
train_inputs = torch.tensor(train_inputs)
validation_inputs = torch.tensor(validation_inputs)
train_labels = torch.tensor(train_labels)
validation_labels = torch.tensor(validation_labels)
train_masks = torch.tensor(train_masks)
validation_masks = torch.tensor(validation_masks)
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)

validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
validation_sampler = SequentialSampler(validation_data)
validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)
train_inputs.shape
train_labels.shape<jupyter_output><empty_output><jupyter_text>Model<jupyter_code># need to download the base uncased model, 420M
# The default download path is ~/.cache
model1 = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=2)
model2 = BertModel.from_pretrained("bert-base-uncased", num_labels=2)
input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
outputs1 = model1(input_ids)
outputs2 = model2(input_ids)
outputs1
encoder_out, text_cls = outputs2
encoder_out.unsqueeze(1)
model1.config
model2.config<jupyter_output><empty_output><jupyter_text>Hyperparamters recommended:- Batch size: 16, 32- Learning rate (Adam): 5e-5, 3e-5, 2e-5- Number of epochs: 2, 3, 4<jupyter_code>param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
     'weight_decay_rate': 0.01},
    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
     'weight_decay_rate': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5)
num_train_optimization_steps = int(len(train_data) / batch_size / 1) * 4
scheduler = WarmupLinearSchedule(optimizer, warmup_steps=100, t_total=num_train_optimization_steps)<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)
train_loss_set = []

# Number of training epochs (authors recommend between 2 and 4)
epochs = 4

# trange is a tqdm wrapper around the normal python range
for _ in trange(epochs, desc="Epoch"):

    # Training

    # Set our model to training mode (as opposed to evaluation mode)
    model.train()
    # Tracking variables
    tr_loss = 0
    nb_tr_examples, nb_tr_steps = 0, 0

    # Train the data for one epoch
    for step, batch in enumerate(train_dataloader):
        # Add batch to GPU
#         batch = tuple(t.to(device) for t in batch)
        # Unpack the inputs from our dataloader
        b_input_ids, b_input_mask, b_labels = batch
        # Clear out the gradients (by default they accumulate)
        optimizer.zero_grad()
        # Forward pass
        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)
        loss = outputs[0]
        train_loss_set.append(loss.item())    
        # Backward pass
        loss.backward()
        # Update parameters and take a step using the computed gradient
        optimizer.step()
        scheduler.step()

        # Update tracking variables
        tr_loss += loss.item()
        nb_tr_examples += b_input_ids.size(0)
        nb_tr_steps += 1

        print("Train loss: {}".format(tr_loss/nb_tr_steps))


    # Validation

    # Put model in evaluation mode to evaluate loss on the validation set
    model.eval()

    # Tracking variables 
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0

    # Evaluate data for one epoch
    for batch in validation_dataloader:
        # Add batch to GPU
#         batch = tuple(t.to(device) for t in batch)
        # Unpack the inputs from our dataloader
        b_input_ids, b_input_mask, b_labels = batch
        # Telling the model not to compute or store gradients, saving memory and speeding up validation
        with torch.no_grad():
          # Forward pass, calculate logit predictions
          logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)

        # Move logits and labels to CPU
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        tmp_eval_accuracy = flat_accuracy(logits, label_ids)

        eval_accuracy += tmp_eval_accuracy
        nb_eval_steps += 1

        print("Validation Accuracy: {}".format(eval_accuracy/nb_eval_steps))
plt.figure(figsize=(15,8))
plt.title("Training loss")
plt.xlabel("Batch")
plt.ylabel("Loss")
plt.plot(train_loss_set)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09703368526897939, 'fraction_numerical': 0.015082956259426848, 'mean_word_length': 4.829304029304029, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '143635', 'license': 'MIT', 'n_tokens_mistral': 2720, 'n_tokens_neox': 2589, 'n_words': 649, 'path': 'Representation/Bert-Fine-Tuning-Pytorch.ipynb', 'repo_name': 'hscspring/YamNLP', 'size': '7965'}