<jupyter_start><jupyter_text>One-timers<jupyter_code>!nvidia-smi
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
!mkdir dataset

!cp /content/drive/MyDrive/Research/triples/data/*.csv dataset
!cp -r /content/drive/MyDrive/Research/triples/HuggingFace dataset
!ls dataset
!pip3 install -q transformers tensorboard_logger seqeval sentencepiece tokenizers sentence_transformers<jupyter_output>Tue Jul 27 12:56:59 2021       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 470.42.01    Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   36C    P0    25W / 300W |      0MiB / 16160MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-------[...]<jupyter_text>Imports<jupyter_code>import os
os.environ['CUDA_LAUNCH_BLOCKING'] = "1" 
import time, torch, random, glob, re, gc, datetime, tokenizers, pdb, errno

import numpy as np
import transformers
import pandas as pd
import torch.nn as nn
import seaborn as sns
import matplotlib.pyplot as plt

from tokenizers import *
from transformers import *
from functools import partial
from pathlib import Path
from tqdm.notebook import tqdm
from torch.nn import functional as F
from itertools import cycle, chain
from torch.utils.data import Dataset, DataLoader, IterableDataset, TensorDataset
# from sklearn.model_selection import GroupKFold
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score
from sklearn.model_selection import train_test_split, RepeatedKFold, KFold
from ast import literal_eval as eval

# from transformers import RobertaForSequenceClassification, RobertaConfig, RobertaTokenizer, RobertaForTokenClassification
import sys
DRIVE_DIR="/content/drive/My Drive/Research/triples/"
sys.path.insert(0, DRIVE_DIR)
from utils import seed_everything, count_params<jupyter_output><empty_output><jupyter_text>Globals and Config<jupyter_code>class Config:
    random_state=2021
    k=5
    device="cuda"
    seed = 2021
    model="bert-base-cased"
    checkpoints=[f"/content/drive/MyDrive/Research/triples/2021-07-24/NER_bert-base-cased_fold-{i+1}_epoch-12_.pt" for i in range(k)]
    pretrained=True
    lowercase = False
    task="NER"
    num_labels=4
    batch_size = 256
    weight_decay =0.001
    max_len=128
    
CP_DIR=Path("/content/drive/MyDrive/Research/triples")
NUM_WORKERS = 2
Config.checkpoints
TRANSFORMERS_DIR=Path("dataset/HuggingFace/")
TRANSFORMERS={
    "roberta-base":{
        "model_config":(RobertaModel, RobertaConfig),
        "tokenizer":RobertaTokenizer,
    },
    "bert-base-cased":{
        "model_config":(BertModel, BertConfig),
        "tokenizer":BertWordPieceTokenizer,
    },
    "bert-base-uncased":{
        "model_config":(BertModel, BertConfig),
        "tokenizer":BertWordPieceTokenizer,
    },
    "albert-base-v2":{
        "model_config":(AlbertModel,AlbertConfig),
        "tokenizer":AlbertTokenizer,
    },
    "gpt2":{
        "model_config":(GPT2Model, GPT2Config),
        "tokenizer":GPT2Tokenizer,
    },
    "distilbert-base-cased":{
        "model_config":(DistilBertModel, DistilBertConfig),
        "tokenizer":DistilBertTokenizer,
    }
}<jupyter_output><empty_output><jupyter_text>Function and Helpers<jupyter_code>def load(model, with_checkpoint=None):
    model=Transformer(model)
    if with_checkpoint:
        checkpoint=torch.load(with_checkpoint, map_location="cpu")
        model.load_state_dict(checkpoint)
        print("Checkpoint loaded!", end="\r")
    return model<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code># class TestDataset(Dataset):
#     def __init__(self, df, tokenizer, tokens, max_len=128, model_name="bert", task="qa"):
#         self.tokenizer = tokenizer
#         self.tokens = tokens
#         self.max_len = max_len
#         try:
#             processed = df.processed.map(eval)
#         except:
#             processed = df.processed.map

#         self.sentences = list(chain.from_iterable(processed))
#         row_number = []
#         for idx, item in enumerate(df.num_sentences.values):
#             row_number.extend([idx] * item)
#         self.row_number = row_number

#     def __len__(self):
#         return len(self.sentences)

#     def __getitem__(self, idx):
#         data = self.sentences[idx]
#         tokenized = self.tokenizer.encode(data)
#         input_ids = tokenized.ids
#         offsets = tokenized.offsets

#         return {
#             "input_ids": torch.tensor(input_ids, dtype=torch.long),
#             "offsets": offsets,
#             "data": data,
#         }
class TestDataset(IterableDataset):
    def __init__(self, df, tokenizer, tokens, max_len=16):
        self.tokenizer=tokenizer
        self.tokens=tokens
        self.max_len=max_len
        try:
            processed=df.processed.map(eval)
        except:
            processed=df.processed
            
        self.sentences=list(chain.from_iterable(processed))
        row_number=[]
        for idx, item in enumerate(df.num_sentences.values):
            row_number.extend([idx]*item)
        self.row_number=row_number
        
        self.ids_=list(range(len(self.sentences)))
        
        self.pad_token=0

        
    def __len__(self):
        """Inaccurate function for necessity"""
        return len(self.sentences)
    
    def pad(self, array, pad_token=-2):
        """Customized padding"""        
        if pad_token==-2:
            pad_token=self.pad_token
            

        end_=len(array)%self.max_len
        if end_>0:
            full_len=((len(array)//self.max_len)+1)*self.max_len
        elif end_==0:
            return array
        else:
            full_len=self.max_len
             
        newArray=np.full(full_len, pad_token)
        end=end_+(len(array)//self.max_len)*self.max_len
        newArray[:end]=array
        return newArray
    
    def chunks(self, list_):

        "for an array longer than the maxlen, this function returns a 2d array"
        l=len(list_)
        n=self.max_len
        if l%n>0:
            N=n*((l//n)+1)
        else:
            N=l-l%n
        for i in range(0, N, n):
            yield np.array(list_[i:i+n], dtype="long")
    
    def getitems(self, idx):

        data=self.sentences[idx]
        tokenized=self.tokenizer.encode(data)
        input_ids=tokenized.ids
        offsets=tokenized.offsets
        offset_start=[i[0] for i in offsets]
        offset_end=[i[1] for i in offsets]
            
#         if len(input_ids)>self.max_len and (len(input_ids)%self.max_len)<(self.max_len*0.5):
#             input_ids=input_ids[:self.max_len]
#             input_ids[-1]=102
            
        if len(input_ids)==self.max_len:
            input_ids=torch.LongTensor([input_ids])
            offset_start=torch.LongTensor([offset_start])
            offset_end=torch.LongTensor([offset_end])
            
        elif len(input_ids)<self.max_len:
            input_ids=self.pad(input_ids)
            offset_start=self.pad(offset_start, -1)
            offset_end=self.pad(offset_end, -1)
            input_ids=torch.LongTensor([input_ids])
            offset_start=torch.LongTensor([offset_start])
            offset_end=torch.LongTensor([offset_end])
            
        else:
            temp1=self.max_len-(len(input_ids)%self.max_len)
            input_ids.extend([0]*temp1)
            offset_start.extend([-1]*temp1)
            offset_end.extend([-1]*temp1)
            input_ids=torch.LongTensor(list(self.chunks(input_ids)))
            offset_start=torch.LongTensor(list(self.chunks(offset_start)))            
            offset_end=torch.LongTensor(list(self.chunks(offset_end)))

        shape=offset_start.shape
        
        for idx_, btch in enumerate(input_ids):
            yield {
                "data":data,
                "input_ids":torch.as_tensor(btch, dtype=torch.long),
                "offsets_start":torch.as_tensor(offset_start[idx_], dtype=torch.long),
                "offsets_end":torch.as_tensor(offset_end[idx_],dtype=torch.long),
                "row_number":torch.as_tensor(self.row_number[idx])
            }

    def getStream(self, ids):
        yield from chain.from_iterable(map(self.getitems, ids))

    def __iter__(self):
        return self.getStream(self.ids_)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class Transformer(nn.Module):

  
    def __init__(self, model, maxlen=128):
        super().__init__()
        self.name = model
        model_type, config_type=TRANSFORMERS[model]['model_config']
        if Config.pretrained:
            self.transformer=model_type.from_pretrained(model, output_hidden_states=True, num_labels=Config.num_labels)
        else:
            config_file=TRANSFORMERS[model]['config']
            config=config_type.from_json_file(config_file)
            config.num_labels=Config.num_labels
            config.output_hidden_states=True
            self.transformer=model_type(config)
            
        self.nb_features = self.transformer.pooler.dense.out_features
        if "roberta" in self.name:
            self.pad_idx=1
        else:
            self.pad_idx=0
        self.logits = nn.Sequential(
            nn.Linear(self.nb_features, self.nb_features),
            nn.Tanh(),
            nn.Linear(self.nb_features, Config.num_labels),
        )

    def forward(self, input_ids, attention_mask=None):        
        hidden_states = self.transformer(
            input_ids,
            attention_mask=(input_ids != self.pad_idx).long(),
        )[-1]

        features = hidden_states[-1]
        logits = torch.sigmoid(self.logits(features))
        
        return logits<jupyter_output><empty_output><jupyter_text>Fitting<jupyter_code>def fit(model,dataset, batch_size, first_time=False):

  """Fits in batches and returns the logits"""
    dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=NUM_WORKERS)

    #just the streaming dataset thing
    if batch_size==256:
      dataloader_len=620
    else:
      for idx, _ in enumerate(dataloader):
        pass
      dataloader_len=idx+1

    model.to(Config.device)
    all_logits=[]
    start_indices=[]
    end_indices=[]
    rows=[]
    texts=[]
    with torch.no_grad():
        with tqdm(total=dataloader_len, desc="inference iterations", unit="batch", position=1, leave=True) as pbar:
            for idx, data in enumerate(dataloader):
              input_ids=data['input_ids']
              logits=model(input_ids=input_ids.to(Config.device))
              all_logits.append(logits.detach().cpu())

              if first_time:
                start=data['offsets_start'].detach().cpu()
                end=data['offsets_end'].detach().cpu()
                row=data['row_number'].detach().cpu()
                text=data['data']


                start_indices.append(start)
                end_indices.append(end)
                rows.append(row)
                texts.append(text)
              pbar.update()

    if Config.device != "cpu":
        torch.cuda.empty_cache()
    gc.collect()
    if first_time:
      all_logits= torch.cat(all_logits, dim=0)
      start_indices= torch.cat(start_indices)
      end_indices=torch.cat(end_indices)
      rows=torch.cat(rows)
      texts=list(chain.from_iterable(texts))
      return all_logits, start_indices, end_indices, rows, texts
    else:
      return torch.cat(all_logits, dim=0)<jupyter_output><empty_output><jupyter_text>K-fold inference<jupyter_code>def k_fold(df, save=True, config=None):

  """Calls fit function for each checkpoint and then ensembles the result."""
    tokenizer = BertWordPieceTokenizer(
                "dataset/HuggingFace/Bert/bert_base_uncased_vocab.txt",
                lowercase=False
            )
    tokens = {
            'cls': tokenizer.token_to_id('[CLS]'),
            'sep': tokenizer.token_to_id('[SEP]'),
            'pad': tokenizer.token_to_id('[PAD]'),
        }

    seed_everything(config.seed)
    checkpointwise_preds=[]
    first_time=True
    for idx, checkpoint in enumerate(Config.checkpoints):

      score = 0
      dataset=TestDataset(df=df, tokenizer=tokenizer, tokens=tokens, max_len=Config.max_len)
      model=load(Config.model, with_checkpoint=checkpoint)
      # if not model.transformer.config.pad_token_id:
      #     model.transformer.config.pad_token_id=tokenizer.eos_token_id

      model.eval()
      if first_time:
        preds, start_indices, end_indices, rows, texts = fit(model, dataset,  batch_size=config.batch_size, first_time=first_time)
        first_time=False
      else:
        preds=fit(model, dataset,  batch_size=config.batch_size, first_time=first_time)
      checkpointwise_preds.append(preds)

      del model, dataset, preds
      if Config.device!="cpu":
          torch.cuda.empty_cache()
      gc.collect()
    preds=torch.mean(torch.stack(checkpointwise_preds), dim=0)
    
    return preds, start_indices, end_indices, rows, texts<jupyter_output><empty_output><jupyter_text>Infer<jupyter_code>df=pd.read_csv("dataset/20NewsGroups.csv")
df["num_sentences"]=df['processed'].map(eval).map(len)
folds=k_fold(df,save=True, config=Config)
def batch_decode(output, offset_s, offset_e, text):
  """Decodes the predictions back to text using the offsets of the tokenizer"""
    all_results=[]
    for batch in range(output.shape[0]):
        encoded_decode=dict()
        if torch.count_nonzero(output[batch]).item()==0:
            continue
        out=output[batch].tolist()
        idx=0
        while idx<len(out):
            if out[idx]!=0:
                idx2=idx
                temp_entity=[]
                predicted_class=out[idx2]

                while (idx2<len(out)) and (out[idx2]==predicted_class):
                    temp_entity.append(idx2)
                    idx2+=1

                encoded_decode[predicted_class]=[(offset_s[batch][i], offset_e[batch][i]) for i in temp_entity]
                
            idx+=1
            

        result=[" "]*3
        for key in encoded_decode.keys():
            res=encoded_decode[key][0]
            res=text[0][res[0].item():res[1].item()+1].strip()
            result[key-1]=res
        if '' in result or " " in result or len(set(result))<3:
            continue
        
        if result not in all_results:
          all_results.append(result)

    return all_results
out=folds[0].argmax(2)
starts=folds[1]
ends=folds[2]
rows=folds[3]
texts=folds[4]
row_wise_results=[]
for i in range(max(rows)+1):    
    temp=np.argwhere(rows==i).flatten()
    out_=out[temp]
    start=starts[temp]
    end=ends[temp]
    texts_=[texts[i] for i in temp]
    row_wise_results.append(batch_decode(out_, start, end, texts_))

df['bert_triples']=row_wise_results
df.head()
df[['filenames','data','bert_triples']].to_csv("/content/drive/MyDrive/Research/triples/data/results.csv", index=False)
df
len(df[df['bert_triples'].apply(str)!="[]"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13398964393019241, 'fraction_numerical': 0.01176244965799399, 'mean_word_length': 2.45037494486105, 'pattern_counts': {'":': 28, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '318454', 'license': 'Apache-2.0', 'n_tokens_mistral': 5119, 'n_tokens_neox': 4946, 'n_words': 806, 'path': 'Colab_Inference_Triples.ipynb', 'repo_name': 'iabd/TriplesExtraction', 'size': '15643'}