<jupyter_start><jupyter_text><jupyter_code>!pip install transformers
from transformers import BertTokenizer, BertForMaskedLM
import torch
import pandas as pd
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForMaskedLM.from_pretrained('bert-base-uncased')
data = pd.read_csv('unredactor.tsv', on_bad_lines='skip',sep='\t', header=None)
data.columns =['author','type','label','text']
data.text[1]
data.shape
train_data = data[data.type=='training']
train_data = train_data.apply(lambda x : x['text'].replace('█'*len(x['label']),x['label']), axis=1).tolist()
inputs = tokenizer(train_data,return_tensors='pt', truncation=True,max_length=50,padding=True)
inputs.keys()
inputs
inputs['labels'] = inputs.input_ids.detach().clone()
inputs.keys()
# create random array of floats in equal dimension to input_ids
rand = torch.rand(inputs.input_ids.shape)

mask_arr = (rand < 0.15) * (inputs.input_ids != 101) * (inputs.input_ids != 102)
mask_arr
inputs.input_ids[2].shape
selection = []

for i in range(inputs.input_ids.shape[0]):
    selection.append(
        torch.flatten(mask_arr[i].nonzero()).tolist()
    )
for i in range(inputs.input_ids.shape[0]):
    inputs.input_ids[i, selection[i]] = 103
inputs.input_ids
class MeditationsDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings
    def __getitem__(self, idx):
        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
    def __len__(self):
        return len(self.encodings.input_ids)
dataset = MeditationsDataset(inputs)
loader = torch.utils.data.DataLoader(dataset, batch_size=16, shuffle=True)
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
# and move our model over to the selected device
model.to(device)
# activate training mode
model.train()
from transformers import AdamW
# initialize optimizer
optim = AdamW(model.parameters(), lr=5e-5)<jupyter_output><empty_output><jupyter_text>Trainer API<jupyter_code>from transformers import TrainingArguments

args = TrainingArguments(
    output_dir='out',
    per_device_train_batch_size=16,
    num_train_epochs=5,

)
from transformers import Trainer

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=dataset
)
trainer.train()
# tokenizer = AutoTokenizer.from_pretrained(model_name)
# model = AutoModelForMaskedLM.from_pretrained(model_name)
# unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)
model.save_pretrained('./')
trainer.save_model('./')
model.eval()
test_token = tokenizer("I'll admit that I was reluctant to see it because from what I knew of [MASK] he was only able to do comedy.",return_tensors='pt', truncation=True,max_length=50,padding=True)
trainer.save_model("./out/model2/")
bert_model = BertForMaskedLM.from_pretrained("./out/model2")
text = "I'll admit that I was reluctant to see it because from what I knew of [MASK] he was only able to do comedy."
from torch.nn import functional as F
input = tokenizer.encode_plus(text, return_tensors = "pt")
mask_index = torch.where(input["input_ids"][0] == tokenizer.mask_token_id)
output = bert_model(**input)
logits = output.logits
softmax = F.softmax(logits, dim = -1)
mask_word = softmax[0, mask_index, :]
top_10 = torch.topk(mask_word, 10, dim = 1)[1][0]
for token in top_10:
   word = tokenizer.decode([token])
   new_sentence = text.replace(tokenizer.mask_token, word)
   print(new_sentence)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1340443931968867, 'fraction_numerical': 0.012107235514557509, 'mean_word_length': 5.995967741935484, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '511713', 'license': 'Apache-2.0', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1165, 'n_words': 255, 'path': 'BERT_MLM.ipynb', 'repo_name': 'harinadh12/Freetime_NLP', 'size': '3469'}