<jupyter_start><jupyter_text>Installing the required packages<jupyter_code>!pip install transformers
!pip install sentencepiece<jupyter_output>Collecting transformers
  Downloading transformers-4.15.0-py3-none-any.whl (3.4 MB)
[K     |████████████████████████████████| 3.4 MB 4.2 MB/s 
[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)
Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)
Collecting pyyaml>=5.1
  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)
[K     |████████████████████████████████| 596 kB 64.7 MB/s 
[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.10.0)
Collecting sacremoses
  Downloading sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)
[K     |████████████████████████████████| 895 kB 61.1 MB/s 
[?25hCollecting huggingface-hub<1.0,>=0.1.0
  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)
[K     |█████████████████████[...]<jupyter_text>Importing required libraries<jupyter_code>import pandas as pd
import os
import torch
from transformers import T5Tokenizer, T5ForConditionalGeneration
from transformers.optimization import  Adafactor 
import time
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Preprocess the data<jupyter_code>import urllib.request
import zipfile
url = 'https://gitlab.com/shimorina/webnlg-dataset/-/archive/master/webnlg-dataset-master.zip?path=release_v3.0/en/train'
urllib.request.urlretrieve(url, 'web.zip')
with zipfile.ZipFile('web.zip', 'r') as zip_ref:
    zip_ref.extractall('web')
import glob
import os
import re
import xml.etree.ElementTree as ET
import pandas as pd
files = glob.glob("/content/web/webnlg-dataset-master-release_v3.0-en-train/release_v3.0/en/train/**/*.xml", recursive=True)
triple_re=re.compile('(\d)triples')
data_dct={}
for file in files:
    tree = ET.parse(file)
    root = tree.getroot()
    triples_num=int(triple_re.findall(file)[0])
    for sub_root in root:
        for ss_root in sub_root:
            strutured_master=[]
            unstructured=[]
            for entry in ss_root:
                unstructured.append(entry.text)
                strutured=[triple.text for triple in entry]
                strutured_master.extend(strutured)
            unstructured=[i for i in unstructured if i.replace('\n','').strip()!='' ]
            strutured_master=strutured_master[-triples_num:]
            strutured_master_str=(' && ').join(strutured_master)
            data_dct[strutured_master_str]=unstructured
mdata_dct={"prefix":[], "input_text":[], "target_text":[]}
for st,unst in data_dct.items():
    for i in unst:
        mdata_dct['prefix'].append('webNLG')
        mdata_dct['input_text'].append(st)
        mdata_dct['target_text'].append(i)


df=pd.DataFrame(mdata_dct)
df.to_csv('webNLG2020_train.csv')
train_df=pd.read_csv('webNLG2020_train.csv', index_col=[0])<jupyter_output><empty_output><jupyter_text>Trimming off a few data points and so that a batch would not leave any remainder, hence some lines of codes can be avoided (Okay, this might be a hackish way of doing it )<jupyter_code>train_df=train_df.iloc[  :35000,:]
train_df=train_df.sample(frac = 1)
batch_size=8
num_of_batches=len(train_df)/batch_size
num_of_epochs=4
num_of_batches=int(num_of_batches)<jupyter_output><empty_output><jupyter_text>Checking for the GPU availability<jupyter_code>if torch.cuda.is_available():
    dev = torch.device("cuda:0") 
    print("Running on the GPU")
else:
    dev = torch.device("cpu")
    print("Running on the CPU")<jupyter_output>Running on the GPU<jupyter_text>Loading the pretrained model and tokenizer<jupyter_code>tokenizer = T5Tokenizer.from_pretrained('t5-base')
model = T5ForConditionalGeneration.from_pretrained('t5-base', return_dict=True)
#moving the model to device(GPU/CPU)
model.to(dev)<jupyter_output><empty_output><jupyter_text>Initializing the Adafactor optimizer with parameter values suggested for t5<jupyter_code>optimizer = Adafactor(
    model.parameters(),
    lr=1e-3,
    eps=(1e-30, 1e-3),
    clip_threshold=1.0,
    decay_rate=-0.8,
    beta1=None,
    weight_decay=0.0,
    relative_step=False,
    scale_parameter=False,
    warmup_init=False
)<jupyter_output><empty_output><jupyter_text><jupyter_code>from IPython.display import HTML, display

def progress(loss,value, max=100):
    return HTML(""" Batch loss :{loss}
        <progress
            value='{value}'
            max='{max}',
            style='width: 100%'
        >
            {value}
        </progress>
    """.format(loss=loss,value=value, max=max))
num_of_epochs=1<jupyter_output><empty_output><jupyter_text>Training the model<jupyter_code># #Sets the module in training mode
# model.train()

# loss_per_10_steps=[]
# for epoch in range(1,num_of_epochs+1):
#   print('Running epoch: {}'.format(epoch))
  
#   running_loss=0

#   out = display(progress(1, num_of_batches+1), display_id=True)
#   for i in range(num_of_batches):
#     inputbatch=[]
#     labelbatch=[]
#     new_df=train_df[i*batch_size:i*batch_size+batch_size]
#     for indx,row in new_df.iterrows():
#       input = 'WebNLG: '+row['input_text']+'</s>' 
#       labels = row['target_text']+'</s>'   
#       inputbatch.append(input)
#       labelbatch.append(labels)
#     inputbatch=tokenizer.batch_encode_plus(inputbatch,padding=True,max_length=400,return_tensors='pt')["input_ids"]
#     labelbatch=tokenizer.batch_encode_plus(labelbatch,padding=True,max_length=400,return_tensors="pt") ["input_ids"]
#     inputbatch=inputbatch.to(dev)
#     labelbatch=labelbatch.to(dev)

#     # clear out the gradients of all Variables 
#     optimizer.zero_grad()

#     # Forward propogation
#     outputs = model(input_ids=inputbatch, labels=labelbatch)
#     loss = outputs.loss
#     loss_num=loss.item()
#     logits = outputs.logits
#     running_loss+=loss_num
#     if i%10 ==0:      
#       loss_per_10_steps.append(loss_num)
#     out.update(progress(loss_num,i, num_of_batches+1))

#     # calculating the gradients
#     loss.backward()

#     #updating the params
#     optimizer.step()
    
#   running_loss=running_loss/int(num_of_batches)
#   print('Epoch: {} , Running loss: {}'.format(epoch,running_loss))
  
#from google.colab import drive
#drive.mount('/content/drive')
#!ls /content/drive/MyDrive
# model_save_name = 'classifier.pt'
# path = F"/content/drive/MyDrive/WebNLG/{model_save_name}" 
# torch.save(model.state_dict(), path)
model_save_name = 'classifier.pt'
path = F"/content/drive/MyDrive/WebNLG/{model_save_name}"
model.load_state_dict(torch.load(path))<jupyter_output><empty_output><jupyter_text>Plotting the loss over time<jupyter_code># import matplotlib.pyplot as plt
   
# steps = [i*100 for i in range(len(loss_per_10_steps))]
  
# plt.plot(steps, loss_per_10_steps)
# plt.title('Loss')
# plt.xlabel('Steps')
# plt.ylabel('Loss')
# plt.show()<jupyter_output><empty_output><jupyter_text>Testing the model<jupyter_code>model.eval()
input_ids = tokenizer.encode("WebNLG: sidharth | hometown | Delhi && sidharth | play |  football </s>", return_tensors="pt")  # Batch size 1
                                    # Subject | relation | Subject && Location | Verb | Object
input_ids=input_ids.to(dev)
outputs = model.generate(input_ids)
tokenizer.decode(outputs[0])<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>torch.save(model.state_dict(),'/content/drive/MyDrive/WebNLG/pytoch_model.bin')
!wget https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-config.json

tokenizer = T5Tokenizer.from_pretrained('t5-base')
model =T5ForConditionalGeneration.from_pretrained('/content/drive/MyDrive/WebNLG/pytoch_model.bin', 
                                                return_dict=True, config='t5-base-config.json')
def generate(text):
    model.eval()
    input_ids = tokenizer.encode("WebNLG:{} </s>".format(text), 
                               return_tensors="pt")  
    outputs = model.generate(input_ids)
    return tokenizer.decode(outputs[0])
generate('Russia | leader | Putin')
generate('Russia | leader | Putin')
generate('Sidharth | profession | Doctor && Sidharth | home_town | Bombay')
generate('Nie_Haisheng | birthDate | 1964-10-13 &8 Nie_Haisheng | occupation | Fighter_pilot')
generate('Bananaman | creator | Steve_Bright && Bananaman | broadcastedBy | BBC')
generate('Bananaman | lastaired | "1986-04-15" && Bananaman | creator | Steve_Bright')
generate('Alan_B._Miller_Hall | currentTenants | Mason_School_of_Business && Alan_B._Miller_Hall | location | Williamsburg')
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive
{'fraction_non_alphanumeric': 0.14445574771108852, 'fraction_numerical': 0.023849892618966882, 'mean_word_length': 4.603546548448385, 'pattern_counts': {'":': 3, '<': 53, '<?xml version=': 0, '>': 55, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '191653', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3357, 'n_tokens_neox': 3174, 'n_words': 521, 'path': 'WebNLG_G2T_1.ipynb', 'repo_name': 'saksham0309/P2_WebNLG2020', 'size': '8847'}