<jupyter_start><jupyter_text>Functions<jupyter_code>def accuracy(out_logits,labels):
    if type(out_logits) != torch.Tensor:
        pred = [val for vals in out_logits for val in vals]
        target = [val for vals in labels for val in vals]
        return np.mean(np.array(pred)==np.array(target))
    else:
        pred = out_logits.argmax(dim=2)
        return torch.mean((pred[labels!=-100] == labels[labels!=-100]).float()).item()

def val_params(model,val_loader):
    temp = model.eval()
    num_batches = 0
    loss_sum = 0
    accuracy_sum =0
    y_true = []
    y_pred = []
    for batch in val_loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        with torch.no_grad():
            outputs = model.forward(input_ids, attention_mask=attention_mask,labels=labels)
        #loss = criterion(outputs.logits.view(-1,2),labels.view(-1))
        loss = outputs[0]
        if len(outputs)!=1:
            pred = outputs[1].argmax(dim=2)
            y_pred = y_pred + pred[labels!=-100].view(-1).tolist()
            y_true = y_true + labels[labels!=-100].view(-1).tolist()
        else:
            labels = list(map(lambda label,mask:label[mask==1].tolist(),labels,attention_mask))
            pred = model.decode(input_ids, attention_mask=attention_mask)
            pred = [val for vals in pred for val in vals]
            labels = [val for vals in labels for val in vals]
            y_pred = y_pred + pred
            y_true = y_true + labels
            
        loss_sum += loss.item()
        num_batches +=1
    f1_micro = f1_score(np.array(y_true).reshape(-1),np.array(y_pred).reshape(-1),average='micro')
    f1_macro = f1_score(np.array(y_true).reshape(-1),np.array(y_pred).reshape(-1),average='macro')
    acc = accuracy_score(np.array(y_true).reshape(-1),np.array(y_pred).reshape(-1))
    return loss_sum/num_batches,acc,f1_micro,f1_macro

def get_ytyp(model,val_loader):
    temp = model.eval()
    y_true = []
    y_pred = []
    for batch in val_loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        with torch.no_grad():
            outputs = model.forward(input_ids, attention_mask=attention_mask,labels=labels)
        #loss = criterion(outputs.logits.view(-1,2),labels.view(-1))
        loss = outputs[0]
        pred = outputs[1].argmax(dim=2)
        
        y_pred = y_pred + pred[labels!=-100].view(-1).tolist()
        y_true = y_true + labels[labels!=-100].view(-1).tolist()
        
    return y_pred,y_true

def get_model_out(model,val_loader):
    temp = model.eval()
    y_true = []
    y_pred = []
    for batch in val_loader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        with torch.no_grad():
            outputs = model.forward(input_ids, attention_mask=attention_mask,labels=labels)
        #loss = criterion(outputs.logits.view(-1,2),labels.view(-1))
        loss = outputs[0]
        pred = outputs[1].argmax(dim=2)
        
        
        y_pred = y_pred + pred[labels!=-100].view(-1).tolist()
        y_true = y_true + labels[labels!=-100].view(-1).tolist()
        loss_sum += loss.item()
        accuracy_sum += accuracy(outputs[1],labels)
        num_batches +=1
    cm = confusion_matrix(np.array(y_true).reshape(-1),np.array(y_pred).reshape(-1))
    return cm


class DefinitionERDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        return item

    def __len__(self):
        return self.encodings['input_ids'].shape[0]
    

def plot_confusion_matrix(cm, classes,
                        normalize=False,
                        title='Confusion matrix',
                        cmap=plt.cm.Blues,
                         figsize = (10,10)):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        cm =np.round(cm,2)
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')
    
    plt.rcParams["figure.figsize"] = figsize
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    
    

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
            horizontalalignment="center",
            color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

def get_dataloader(dir_path,tokenizer, batch_size=32):
    x,y,tags = pp.get_data_slt(dir_path)
    encodings = tokenizer(x,is_split_into_words=True,  padding=True, truncation=True,return_tensors="pt")
    dataset = DefinitionDataset(encodings,y)
    data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
    return data_loader

def get_encodings(text,tags,tokenizer):
    encodings  = tokenizer(text, is_split_into_words=True, return_offsets_mapping=True, padding=True, truncation=True,return_tensors='pt')
    seq_len = encodings.input_ids.shape[1]
    labels = [[lab2id[tok] for tok in seq] for seq in tags]
    enc_labels = []
    input_ids = []
    out_enc = {}
    for off_map_seq,lab_seq in zip(encodings.offset_mapping,labels):
        tags = np.ones(seq_len,dtype=int) * -100
        
        
        tags[(off_map_seq[:,0] == 0) & (off_map_seq[:,1] != 0)]= lab_seq
        
        enc_labels.append(tags)
        

    encodings['labels'] = torch.tensor(enc_labels)  
    
    return encodings



def get_encodings_crf(text,tags,tokenizer):
    encodings  = tokenizer(text, is_split_into_words=True, return_offsets_mapping=True, padding=True, truncation=True,return_tensors='pt')
    seq_len = encodings.input_ids.shape[1]
    labels = [[lab2id[tok] for tok in seq] for seq in tags]
    enc_labels = []
    input_ids = []
    out_enc = {}
    for off_map_seq,lab_seq,in_id in zip(encodings.offset_mapping,labels,encodings.input_ids):
        inp_id = [0]*seq_len
        tags = [lab2id[' O']]*seq_len
        in_id_nopad = in_id[off_map_seq[:,0]==0][in_id[off_map_seq[:,0]==0]!=0].tolist()
        inp_id[:len(in_id_nopad)]=in_id_nopad
        tags[1:(len(lab_seq)+1)]= lab_seq
        
        input_ids.append(inp_id)
        enc_labels.append(tags)
        

    out_enc['labels'] = torch.tensor(enc_labels) 
    out_enc['input_ids'] = torch.tensor(input_ids) 
    out_enc['attention_mask']=torch.logical_and(out_enc['input_ids']!=0 , out_enc['input_ids']!=102).type(torch.ByteTensor)
    
    return out_enc

!nvidia-smi
!ps aux grep 3165672<jupyter_output>USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root        1613  3.0  0.0      0     0 ?        R    Jan29 414:53 [nv_queue]
root        1977  3.8  0.0      0     0 ?        R    Jan29 520:53 [nv_queue]
gsinha1   576946 44.0  0.0  11716  3604 pts/16   Rs+  19:31   0:00 ps aux grep 3<jupyter_text>Setup<jupyter_code>device = torch.device('cuda:2') if torch.cuda.is_available() else torch.device('cpu')
torch.tensor([1,2]).to(device)
dir_path = '../data/deft_files/train/'
dir_path_val = '../data/deft_files/dev/'
model_name = 'bert-large-cased'  #  "facebook/bart-large" 'roberta-base'
Model = BertForTokenClassification
ModelTokenizer = BertTokenizerFast #BertTokenizerFast
path_to_save = "../model/berta_ner/"<jupyter_output><empty_output><jupyter_text>Data Processing<jupyter_code>texts_train,_,tags_train = pp.get_data_slt(dir_path)
texts_val,_,tags_val = pp.get_data_slt(dir_path_val)
unique_tags = {}
for seq in tags_train:
    for tok in seq:
        if tok not in unique_tags.keys():
            unique_tags[tok] = 1
        else:
            unique_tags[tok] += 1
            
    
id2lab = sorted(list(unique_tags.keys()))
classes = id2lab.copy()
id2lab = {val:key for val,key in enumerate(id2lab)}
id2lab[-100] = 'other'
lab2id = {key:val for val,key in id2lab.items()}
lab2id
weights = [1]*len(id2lab)
for lab,num in unique_tags.items():
    weights[lab2id[lab]] = 1000/num
weight = torch.tensor(weights).to(device)
weight
tokenizer = ModelTokenizer.from_pretrained(model_name,add_prefix_space=True)
train_encodings = get_encodings(texts_train,tags_train,tokenizer)
val_encodings = get_encodings(texts_val,tags_val,tokenizer)
i=79
df = pd.DataFrame({
    'tokens':tokenizer.convert_ids_to_tokens(val_encodings['input_ids'][i].tolist()),
    'label':list(map(lambda x: id2lab[x],val_encodings['labels'][i].tolist()))
})

df.head(50)
train_dataset = DefinitionERDataset(train_encodings)
val_dataset = DefinitionERDataset(val_encodings)
BATCH_SIZE = 32
train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class DefinitionExtraction(nn.Module):
    def __init__(self,num_labels):
        super(DefinitionExtraction, self).__init__()
        self.bert = BertModel.from_pretrained('bert-large-cased')
        self.dense = nn.Linear(1024,1024)
        self.classification = nn.Linear(1024,num_labels)
        self.tanh = nn.Tanh()
        self.dropout = nn.Dropout(p=0.1)
        self.crf = CRF(num_labels,batch_first=True)
        self.criterion = nn.CrossEntropyLoss()
    
    def forward(self,input_ids,attention_mask=None,labels=None):
        x = self.bert(input_ids.long())
        x = self.tanh(self.dense(x.last_hidden_state))
        x = self.dropout(x)
        logits = self.classification(x)
        loss = self.crf.forward(logits,labels,attention_mask)
        return [loss]
    
    def decode(self,input_ids,attention_mask=None):
        x = self.bert(input_ids.long())
        x = self.tanh(self.dense(x.last_hidden_state))
        x = self.dropout(x)
        logits = self.classification(x)
        output = self.crf.decode(logits,attention_mask)
        return output
    
    def call(self,input_ids,attention_mask=None,labels=None):
        return self.forward(input_ids,attention_mask,labels)
#model = DefinitionExtraction(len(id2lab)).to(device)
model = Model.from_pretrained(model_name,num_labels=len(id2lab)).to(device)
#model = nn.DataParallel(model,device_ids=[0,1,3],output_device=1)<jupyter_output>Some weights of the model checkpoint at bert-large-cased were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']
- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForTokenClassification were not initialized from the model checkpoint at bert-large-c[...]<jupyter_text>Training<jupyter_code>EPOCHS=12
optim = AdamW(model.parameters(), lr=1e-5)
sched = get_cosine_schedule_with_warmup(optim,
                                        num_warmup_steps=100,
                                        num_training_steps=(len(tags_train)/BATCH_SIZE)*EPOCHS)
prev_val_acc = -1
temp = model.train()
for epoch in range(EPOCHS):
    loss_sum = 0
    accuracy_sum = 0
    num_batch = 0
    pbar = tqdm.tqdm(train_loader)
    for batch in pbar:
        optim.zero_grad()
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)
        labels = batch['labels'].to(device)
        outputs = model(input_ids, attention_mask=attention_mask,labels=labels)
        loss = outputs[0]
        loss.backward()
        optim.step()
        sched.step()
        loss_sum += loss.item()
        if len(outputs)==1:
            out = model.decode(input_ids, attention_mask=attention_mask)
            outputs.append(out)
            labels = list(map(lambda label,mask:label[mask==1].tolist(),labels,attention_mask))
        accuracy_sum += accuracy(outputs[1],labels)
        num_batch+=1
        pbar.set_description("Epoch: %s, Train loss: %f, Train accuracy: %f"%(epoch,
                                                                              loss_sum/num_batch,
                                                                              accuracy_sum/num_batch))
    val_metric = val_params(model,val_loader)
    sys.stdout.write("         Val loss: %f, Val accuracy: %f, Val f1-micro: %f, Val f1-macro: %f"%val_metric)
    sys.stdout.flush()
    
    #Breaking criteria
    if prev_val_acc > val_metric[2]:
        pass
    
    prev_val_acc = val_metric[2]
    
    #saving model checkpoint
    model.save_pretrained(path_to_save+str(epoch))
    tokenizer.save_pretrained(path_to_save+str(epoch))
val_metric = val_params(model,val_loader)
val_metric<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>tokenizer = ModelTokenizer.from_pretrained(path_to_save+str(0))
model = Model.from_pretrained(path_to_save+str(0)).to(device)
temp = model.eval()
yt,yp = get_ytyp(model,val_loader)
yt = [id2lab[idx] for idx in yt]
yp = [id2lab[idx] for idx in yp]
classes = sorted(list(set(yt+yp)))
cm = confusion_matrix(yt,yp)
plot_confusion_matrix(cm,classes,normalize=True,figsize=(15,15))
import spacy
from nltk.tokenize import word_tokenize
nlp = spacy.load("en_core_web_sm")
#nltk.download('punkt')
text = "SWOT stands for Strengths, Weaknesses, Opportunities, and Threats, and so a SWOT Analysis is a technique for assessing these four aspects of your business."

def get_individual_predictions(i,text=None):
    if text is None:
        custom = False
    else:
        cutome=True

    split_text = texts_val[i]
    tags = tags_val[i]
    if custom:
        split_text = word_tokenize(text)
        tags = [' O']*len(split_text)
    enc = get_encodings([split_text],[tags],tokenizer)
    lab = enc.labels[0]
    with torch.no_grad():
        output = model(enc.input_ids.to(device),enc.attention_mask.to(device))
    pred_out = output[0].argmax(dim=2)[0]
    pred_out=pred_out[lab!=-100]
    lab = lab[lab!=-100]
    lab = [id2lab[idx] for idx in lab.tolist()]
    pred_out = [id2lab[idx] for idx in pred_out.tolist()]

    res = pd.DataFrame({
        'texts':split_text,
         'targ': lab,
        'pred':pred_out

    })
    return res<jupyter_output><empty_output><jupyter_text>Preliminary Results<jupyter_code>res_df = get_individual_predictions(30)
res_df
res_df = get_individual_predictions(1)
res_df
res_df = get_individual_predictions(5)
res_df
res_df = get_individual_predictions(9)
res_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12540275808738238, 'fraction_numerical': 0.01778579713880655, 'mean_word_length': 3.31682892906815, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '148147', 'license': 'MIT', 'n_tokens_mistral': 5450, 'n_tokens_neox': 5255, 'n_words': 929, 'path': 'src/training_ner.ipynb', 'repo_name': 'sgauravm/WordDefinitionExtraction', 'size': '15518'}