<jupyter_start><jupyter_text><jupyter_code># Check GPU availability
!nvidia-smi
'''Importing necessary modules
'''
import numpy as np
import torch
from torch.utils.data import DataLoader, Dataset, TensorDataset
import sys
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import data
import torch.optim as optim
from torch.utils.tensorboard import SummaryWriter

sys.path.append('./Trainers/')
sys.path.append('./Dataloaders/')
sys.path.append('./utils/')

from dataloader import mydataset, create_prime_dict 
'''
For BERT
'''
from transformers import BertForSequenceClassification, AdamW, BertConfig
from transformers import get_linear_schedule_with_warmup
from trainer_BERT import train, test_classify
# Select gpu
device = torch.device('cuda:7' if torch.cuda.is_available() else 'cpu')
print(device)<jupyter_output>cuda:7<jupyter_text>**Dataloading Scheme**<jupyter_code>trainlist = 'add path of training list'
validlist = 'add path of validation list'
prime_dict = create_prime_dict(trainlist)
'''
Train Dataloader
''' 
train_dataset = mydataset(trainlist, prime_dict, name='train')          
train_dataloader = data.DataLoader(train_dataset, shuffle= True, batch_size = 32, num_workers=16,pin_memory=True)


'''
Validation Dataloader
''' 
validation_dataset = mydataset(validlist, prime_dict, name='valid')         
validation_dataloader = data.DataLoader(validation_dataset, shuffle=False, batch_size = 32, num_workers=16,pin_memory=True)<jupyter_output><empty_output><jupyter_text>**Model Definition**<jupyter_code>'''
Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top.
''' 
model = BertForSequenceClassification.from_pretrained(
    "bert-base-uncased", #12-layer BERT model, with an uncased vocab.
    num_labels = 2, #Binary classification
    output_attentions = False, # Whether the model returns attentions weights.
    output_hidden_states = False, # Whether the model returns all hidden-states.
)

model = nn.DataParallel(model,device_ids=[7]).to(device)
model<jupyter_output><empty_output><jupyter_text>**Hyperparameters**<jupyter_code>'''
Loss Function
'''
criterion = nn.CrossEntropyLoss()


'''
Optimizer
'''
optimizer = AdamW(model.parameters(), lr = 2e-5, eps = 1e-8)

'''
Number of training epochs. The BERT authors recommend between 2 and 4. We chose to run for 4, but we'll see later that this may be over-fitting the training data.
'''
num_Epochs = 10


'''
Create the learning rate scheduler.
Total number of training steps is [number of batches] x [number of epochs].
Note that this is not the same as the number of training samples.
'''
total_steps = len(train_dataloader) * num_Epochs
lr_scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 0,  num_training_steps = total_steps)
modelname = 'BERT_basic'
modelpath = './saved_model_checkpoints/'+modelname
writer = SummaryWriter(modelname)

train(model, train_dataloader, validation_dataloader, criterion, optimizer, lr_scheduler, modelpath, writer, device, epochs = num_Epochs)

writer.flush()
writer.close()

'''
Load saved model from checkpoint  #####
'''
model, optimizer, lr_scheduler, train_loss, v_loss, v_acc, epoch = load(modelpath, model, optimizer, lr_scheduler)

model.module.classifier<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10392749244712991, 'fraction_numerical': 0.0072507552870090634, 'mean_word_length': 6.374164810690423, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '197210', 'license': 'CC0-1.0', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1041, 'n_words': 282, 'path': 'Code/Experiments/BERT.ipynb', 'repo_name': 'rezacsedu/Detection-of-Hate-Speech-in-Multimodal-Memes', 'size': '3310'}