<jupyter_start><jupyter_text>Load BERT<jupyter_code>model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')<jupyter_output><empty_output><jupyter_text>Freeze encoder Use only weights of head layers<jupyter_code># for param in model.base_model.parameters():
#     param.requires_grad = False<jupyter_output><empty_output><jupyter_text>Generate Encodings<jupyter_code>train_text = train_data.cleaned.tolist()
valid_text = valid_data.cleaned.tolist()
train_encodings = tokenizer(train_text,truncation=True,padding=True)
valid_encodings = tokenizer(valid_text,truncation=True,padding=True)
class OLIdDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)
train_dataset = OLIdDataset(train_encodings,train_data.label.tolist())
valid_dataset = OLIdDataset(valid_encodings,valid_data.label.tolist())
def compute_metrics(pred):
    labels = pred.label_ids
    preds = pred.predictions.argmax(-1)
    f1m = f1_score(labels,preds,average='macro')
    acc = accuracy_score(labels, preds)
    return {
        'accuracy': acc,
        'macro f1': f1m
    }

training_args = TrainingArguments(
    output_dir='./results',
    num_train_epochs=20,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
)

trainer = Trainer(
    model=model,
    args=training_args,
    compute_metrics=compute_metrics,
    train_dataset=train_dataset,
    eval_dataset=valid_dataset
)
trainer.train()
trainer.evaluate()
%load_ext tensorboard
%tensorboard --logdir logs<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11066599899849774, 'fraction_numerical': 0.008512769153730596, 'mean_word_length': 5.24375, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '492639', 'license': 'MIT', 'n_tokens_mistral': 701, 'n_tokens_neox': 687, 'n_words': 99, 'path': 'Experiments/DistilBERT_Classification.ipynb', 'repo_name': 'Ashwin-19/OffensEval-2019', 'size': '1997'}