<jupyter_start><jupyter_text>Fine tuning BERTBased on the tutorial from http://mccormickml.com/2019/07/22/BERT-fine-tuning/11-using-colab-gpu-for-training<jupyter_code>import tensorflow as tf
device_name = tf.test.gpu_device_name()

if device_name == '/device:GPU:0':
    print('Found GPU at: {}'.format(device_name))
else:
    raise SystemError('GPU device not found')
import torch
if torch.cuda.is_available():
  device = torch.device('cuda')
  print("There are %d GPUs available" % torch.cuda.device_count())
  print(' We will use the GPU:', torch.cuda.get_device_name(0))
else:
  print(' No GPU available, using CPU instead')
  device = torch.device('cpu')
is_colab = False
if 'google.colab' in str(get_ipython()):
  print('Running on CoLab')
  is_colab = True
else:
  print('Not running on CoLab')
if is_colab:
    %pwd
    from google.colab import drive
    drive.mount('/content/gdrive', force_remount=True)
    print("Mounted Drive successfully")
    %cd /content
    %cd /content/gdrive/My\ Drive/abhip-models/
    print("Should be in the drive folder of the project, check previous printed line to confirm")
!pip install transformers<jupyter_output>Collecting transformers
[?25l  Downloading https://files.pythonhosted.org/packages/d8/f4/9f93f06dd2c57c7cd7aa515ffbf9fcfd8a084b92285732289f4a5696dd91/transformers-3.2.0-py3-none-any.whl (1.0MB)
[K     |████████████████████████████████| 1.0MB 5.3MB/s 
[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)
Collecting sacremoses
[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)
[K     |████████████████████████████████| 890kB 38.0MB/s 
[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)
Collecting sentencepiece!=0.1.92
[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c[...]<jupyter_text>I don't expect to have consistent runtime for the work and would be useful to keep them in peristent storage in My Drive<jupyter_code>%pwd
!pip install wget

import wget
import os

print('Downloading the dataset')
url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'

if not os.path.exists('./cola_public_1.1.zip'):
  try: 
    wget.download(url, './cola_public_1.1.zip')
  except Exception as e:
    print("Failed to use py wget")
    print(e)

if not os.path.exists('./cola_public/'):
    !unzip cola_public_1.1.zip
import pandas as pd
df = pd.read_csv("./cola_public/raw/in_domain_train.tsv",  delimiter='\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])
print('Number of training sentences: {:,}\n'.format(df.shape[0]))
df.sample(10)
df.loc[df.label == 0].sample(5)[['sentence', 'label']]
sentences = df.sentence.values
labels = df.label.values
from transformers import BertTokenizer
# Load the BERT tokenizer.
print('Loading BERT tokenizer...')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
print('Original: ', sentences[0])

print('Tokenized: ', tokenizer.tokenize(sentences[0]))

print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])))
max_len = 0
for sent in sentences:
  input_ids = tokenizer.encode(sent, add_special_tokens=True)

  max_len = max(max_len, len(input_ids))
print("Max length:", max_len)
if  (not os.path.exists('./input_ids.pt')) or (not os.path.exists('./attention_masks.pt')) or (not os.path.exists('./labels.pt')):
  input_ids =[]
  attention_masks = []

  for sent in sentences:
    encoded_dict = tokenizer.encode_plus(
        sent,
        add_special_tokens = True, 
        max_length = 64,
        pad_to_max_length = True,
        return_attention_mask = True, 
        return_tensors = 'pt'
    )

    input_ids.append(encoded_dict['input_ids'])
    
    attention_masks.append(encoded_dict['attention_mask'])

  input_ids = torch.cat(input_ids, dim=0)
  attention_masks = torch.cat(attention_masks, dim=0)
  labels = torch.tensor(labels)

  torch.save(input_ids, 'input_ids.pt')
  torch.save(attention_masks, 'attention_masks.pt')
  torch.save(labels, 'labels.pt')


  print('Original:  ', sentences[0])
  print('Token IDs: ', input_ids[0])
else:
  print("Loading the tensors from Drive")
  input_ids = torch.load('./input_ids.pt')
  attention_masks = torch.load('./attention_masks.pt')
  labels = torch.load('./labels.pt')
from torch.utils.data import TensorDataset, random_split

# Combine the training inputs into a TensorDataset.
dataset = TensorDataset(input_ids, attention_masks, labels)

# Create a 90-10 train-validation split.

# Calculate the number of samples to include in each set.
train_size = int(0.9 * len(dataset))
val_size = len(dataset) - train_size

# Divide the dataset by randomly selecting samples.
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])

print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))

from torch.utils.data import DataLoader, RandomSampler, SequentialSampler

batch_size = 32

train_dataloader = DataLoader(
    train_dataset, 
    sampler = RandomSampler(train_dataset),
    batch_size = batch_size
)

validation_dataloader = DataLoader(
    val_dataset,
    sampler = SequentialSampler(val_dataset),
    batch_size = batch_size
)


from transformers import BertForSequenceClassification, AdamW, BertConfig

model = BertForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels = 2,
    output_attentions = False, 
    output_hidden_states = False
)

model.cuda()
params = list(model.named_parameters())

print('The BERT model has {:} different named parameters.\n'.format(len(params)))

print('==== Embedding Layer ====\n')

for p in params[0:5]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

print('\n==== First Transformer ====\n')

for p in params[5:21]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

print('\n==== Output Layer ====\n')

for p in params[-4:]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

optimizer = AdamW(
    model.parameters(),
    lr  = 2e-5,
    eps =  1e-8
)
from transformers import get_linear_schedule_with_warmup
epochs = 4

total_steps = len(train_dataloader) * epochs;

scheduler = get_linear_schedule_with_warmup(optimizer, 
                                           num_warmup_steps =0,
                                           num_training_steps = total_steps)
import numpy as np
def flat_accuracy(preds, labels):
  pred_flat = np.argmax(preds, axis = 1).flatten()
  labels_flat = labels.flatten()
  return np.sum(pred_flat == labels_flat) / len(labels_flat)
import time
import datetime

def format_time(elapsed):
  elapsed_rounded = int(round(elapsed))
  return str(datetime.timedelta(seconds=elapsed_rounded))
# Training Loop
import random
import numpy as np

seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)


training_stats = []

total_t0 = time.time()

for epoch_i in range(0, epochs):
  print("")
  print('======== Epoch {:} / {:} ========'.format(epoch_i +1, epochs))
  print('Training...')


  t0 = time.time()

  total_train_loss = 0
  # Put it into training mode
  model.train()

  for step, batch in enumerate(train_dataloader):
    if step%40 == 0 and not step == 0:
      elapsed = format_time(time.time() - t0)

      print('Batch {:>5}  of  {:>5}.   Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))

      b_input_ids = batch[0].to(device)
      b_input_mask = batch[1].to(device)
      b_labels = batch[2].to(device)

      model.zero_grad()

      loss, logits = model(b_input_ids,
                           token_type_ids = None,
                           attention_mask = b_input_mask, 
                           labels = b_labels)

      total_train_loss += loss.item()

      loss.backward()

      torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

      optimizer.step()

      scheduler.step()

  avg_train_loss = total_train_loss / len(train_dataloader)

  training_time = format_time(time.time() - t0)

  print("")
  print(" Average training loss {0:.2f}".format(avg_train_loss))
  print(" Training epoch took: {:}".format(training_time))

  print("")
  print("Running Validation...")


  t0 = time.time()

  model.eval()

  total_eval_accuracy = 0
  total_eval_loss = 0
  nb_eval_steps = 0

  for batch in validation_dataloader:
    b_input_ids = batch[0].to(device)
    b_input_mask = batch[1].to(device)
    b_labels = batch[2].to(device)

    with torch.no_grad():
      (loss, logits) = model(b_input_ids,
                             token_type_ids = None, 
                             attention_mask = b_input_mask,
                             labels = b_labels)
      
    total_eval_loss =+ loss.item()

    logits = logits.detach().cpu().numpy()
    label_ids = b_labels.to('cpu').numpy()

    total_eval_accuracy += flat_accuracy(logits, label_ids)
  

  avg_val_accuracy = total_eval_accuracy/ len(validation_dataloader)
  print(" Accuracy: {0:.2f}".format(avg_val_accuracy))

  avg_val_loss = total_eval_loss/ len(validation_dataloader)
  validation_time = format_time(time.time() - t0)
  print(" Validation Loss: {0:.2f}".format(avg_val_loss))
  print(" Validation took: {:}".format(validation_time))

  training_stats.append(
      {
          'epoch': epoch_i + 1,
       'Training Loss': avg_train_loss,
       'Valid. Loss': avg_val_loss,
       'Valid. Accur.': avg_val_accuracy,
       'Training Time': training_time,
       'Validation Time': validation_time
      }
  )

print("")
print("Training complete!")

print("Total training took {:} (h:mm:ss)".format(format_time(time.time() - total_t0)))

import pandas as pd

pd.set_option('precision', 2)

df_stats = pd.DataFrame(data=training_stats)

df_stats = df_stats.set_index('epoch')

df_stats
import matplotlib.pyplot as plt
% matplotlib inline

import seaborn as sns

sns.set(style='darkgrid')

sns.set(font_scale = 1.5)
plt.rcParams["figure.figsize"] = (12,6)

plt.plot(df_stats['Training Loss'], 'b-o', label="Training")
plt.plot(df_stats['Valid. Loss'], 'g-o', label="Validation")

plt.title("Training & Validation Loss")
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.legend()
plt.xticks([1,2,3,4])

plt.show()
import pandas as pd

df = pd.read_csv("./cola_public/raw/out_of_domain_dev.tsv", delimiter = "\t", header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])

print('Number of test sentences: {:,}\n'.format(df.shape[0]))

sentences = df.sentence.values
labels = df.label.values

input_ids = []
attention_masks = []


for sent in sentences:
  encoded_dict = tokenizer.encode_plus(
      sent, 
      add_special_tokens = True,
      max_length = 64, 
      pad_to_max_length = True,
      return_attention_mask = True,
      return_tensors = 'pt'
      )
  
  input_ids.append(encoded_dict['input_ids'])

  attention_masks.append(encoded_dict['attention_mask'])

input_ids = torch.cat(input_ids, dim = 0)
attention_masks = torch.cat(attention_masks, dim=0)
labels = torch.tensor(labels)

batch_size = 32

prediction_data = TensorDataset(input_ids, attention_masks, labels)
prediction_sampler = SequentialSampler(prediction_data)
prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size = batch_size)
print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))
model.eval()

predictions, true_labels =[], []

for batch in prediction_dataloader:
  batch = tuple(t.to(device) for t in batch)

  b_input_ids, b_input_mask, b_labels = batch

  with torch.no_grad():
    outputs = model(b_input_ids, token_type_ids= None, attention_mask = b_input_mask)

  logits = outputs[0]

  logits = logits.detach().cpu().numpy()
  label_ids = b_labels.to('cpu').numpy()

  predictions.append(logits)
  true_labels.append(label_ids)


print('DONE.')
print(len(predictions))
print('Positive samples: %d of %d (%.2f%%)' % (df.label.sum(), len(df.label), (df.label.sum())/ len(df.label) * 100))
from sklearn.metrics import matthews_corrcoef

matthews_set = []

print('Calculating Mattews Corr. Coef. for each batch...')

for i in range(len(true_labels)):
  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()
  matthews = matthews_corrcoef(true_labels[i], pred_labels_i)
  matthews_set.append(matthews)

print(matthews_set)
ax = sns.barplot(x = list(range(len(matthews_set))), y = matthews_set, ci = )

plt.title(' MCC Score per batch')
plt.ylabel('MCC Score (-1 to +1)')
plt.xlabel('Batch #')

plt.show()
 
flat_predictions = np.concatenate(predictions, axis=0)

# For each sample, pick the label (0 or 1) with the higher score.
flat_predictions = np.argmax(flat_predictions, axis=1).flatten()

# Combine the correct labels for each batch into a single list.
flat_true_labels = np.concatenate(true_labels, axis=0)

# Calculate the MCC
mcc = matthews_corrcoef(flat_true_labels, flat_predictions)

print('Total MCC: %.3f' % mcc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14351294400963274, 'fraction_numerical': 0.024006622516556293, 'mean_word_length': 4.277601270849881, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '612578', 'license': 'MIT', 'n_tokens_mistral': 4984, 'n_tokens_neox': 4679, 'n_words': 950, 'path': 'Fine_tuning_BERT-TextClassification.ipynb', 'repo_name': 'Abhishek-P/bert-trials', 'size': '13288'}