<jupyter_start><jupyter_text>**SECTION SCAFFOLD DATA REVIEW**<jupyter_code>p=[]
with jsonlines.open(sec_sc) as f:
    for line in f.iter():
        p.append(line)
line.keys()
data = pd.DataFrame(p)
data
data['section_name'].value_counts()
data[['citing_paper_id','cleaned_cite_text','section_title','citation_id','is_citation']]
data['section_name'].value_counts()
sum(data['is_citation']==False)
data.info()
data.describe()
data.head()
data.head()['cleaned_cite_text'][0]
sum(data['text']==data['cleaned_cite_text'])
sum(data['section_name']==data['section_title'])<jupyter_output><empty_output><jupyter_text>**CITATION WORTHINESS SCAFFOLD DATA REVIEW**<jupyter_code>q=[]
with jsonlines.open(cit_sc) as f:
    for line in f.iter():
        q.append(line)
line.keys()
datac = pd.DataFrame(q)
datac['is_citation'].value_counts()
datac[['text','citing_paper_id','cleaned_cite_text','citation_id','is_citation']]
datac.info()
sum(datac['text']!=datac['cleaned_cite_text'])
d=datac.loc[datac['text']!=datac['cleaned_cite_text']]
d['text'][5]
d['cleaned_cite_text'][5]
datac.head()['text'][4]
datac.head()['cleaned_cite_text'][4]
datac.describe()<jupyter_output><empty_output><jupyter_text>**MAIN TASK DATA REVIEW**<jupyter_code>tm=[]
with jsonlines.open(train_main) as f:
    for line in f.iter():
        tm.append(line)
pdm = pd.DataFrame(tm)
pdm['string'][6540]
pdm['label'].value_counts()
pdm.info()
pdm.describe()<jupyter_output><empty_output><jupyter_text>**CITED PAPER TITLE SCAFFOLD**<jupyter_code>pdtr = pd.read_csv('../input/pride-data/train.csv')
pdte = pd.read_csv('../input/pride-data/test.csv')
pdtr
pdtr['citation_context'][0]
x_tit = pdtr['cited_title'].values.tolist()
len(x_tit)
yc = pdtr['citation_class_label'].values.tolist()
label={0:0,1:2,2:1,3:0,4:0,5:1}
y_tit = list(map(lambda t : label[t],yc))
len(y_tit)
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x_tit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),x_tit))
y_tit = torch.tensor(y_tit)
x_tit = torch.tensor(x_tit)
print(x_tit.shape)
print(y_tit.shape)<jupyter_output><empty_output><jupyter_text>**ACL DATA LOADING.....**<jupyter_code>p=[]
with jsonlines.open(acl_sec_sc) as f:
    for line in f.iter():
        p.append(line)
acl_secdata = pd.DataFrame(p)
acl_secdata
acl_secdata.info()
q=[]
with jsonlines.open(acl_cit_sc) as f:
    for line in f.iter():
        q.append(line)
acl_citdata = pd.DataFrame(q)
acl_citdata
acl_citdata.info()
tm=[]
with jsonlines.open(acl_train_main) as f:
    for line in f.iter():
        tm.append(line)
acl_pdm = pd.DataFrame(tm)
acl_pdm
acl_pdm.info()<jupyter_output><empty_output><jupyter_text>**DATA PREP**<jupyter_code>def remove_idx(l,max_len): # max_len = len of tokens criteria above which to remove
    lent = []
    idx_remove = []
    for i in l:
        lent.append(len(tokenizer.encode(i, padding=True, truncation=True, return_tensors="pt")[0]))
    for i in range(len(lent)):
        if(lent[i]>max_len):
            idx_remove.append(i)
    return idx_remove
# returns balanced dataset.....
def class_div(X,Y,instances,num_class):
    inst_per_class = instances//num_class
    rem = instances%num_class
    x_out=[]
    y_out=[]
    for i in range(num_class):
        xcl = [x for x,y in zip(X,Y) if y==i]
        ycl = [y for y in Y if y==i]
        print('xcl length : ',len(xcl))
        if(rem):
            x_out += xcl[:inst_per_class+1]
            y_out += ycl[:inst_per_class+1]
            rem -= 1
            print('x_out length : ',len(x_out))
        else:
            x_out += xcl[:inst_per_class]
            y_out += ycl[:inst_per_class]
            print('x_out length : ',len(x_out))
    return (x_out,y_out)
import re
def fun(i):
    pat = ' '.join(re.split(r'\[[^\[\]]*\]' ,i))
    pat = ' '.join(re.split(r'\([^\[\]\(\)]*\)' ,pat))
    return pat
citm=pdm['string'].values.tolist()
citm = [fun(i) for i in citm]
pdm['processed_text'] = citm
len(citm)
yc = pdm['label'].values.tolist()
label={'background':0,'method':1,'result':2}
y = list(map(lambda t : label[t],yc))
len(y)
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
citm_idx_remove_300 = remove_idx(citm,300)
for i in sorted(citm_idx_remove_300, reverse = True):  
    del citm[i]
    del y[i]
print(f'removed {len(citm_idx_remove_300)} instances')
len(y)
x_cit = datac['cleaned_cite_text'].values.tolist()
x_cit = [fun(i) for i in x_cit]
datac['processed_text'] = x_cit
len(x_cit)
ycit = datac['is_citation'].values.tolist()
ycit_ie = list(map(lambda x : int(x),ycit))
len(ycit_ie)
xcit_idx_remove_300 = remove_idx(x_cit,300)
for i in sorted(xcit_idx_remove_300, reverse = True):  
    del x_cit[i]
    del ycit_ie[i]
print(f'removed {len(xcit_idx_remove_300)} instances')
x_sec = data['text'].values.tolist()
x_sec = [fun(i) for i in x_sec]
data['processed_text'] = x_sec
len(x_sec)
y_sec = data['section_name'].values.tolist()
labels={'introduction':0,'related work':1,'method':2,'experiments':3,'conclusion':4}
ysec_ie = list(map(lambda t : labels[t],y_sec))
len(ysec_ie)
xsec_idx_remove_300 = remove_idx(x_sec,300)
for i in sorted(xsec_idx_remove_300, reverse = True):  
    del x_sec[i]
    del ysec_ie[i]
print(f'removed {len(xsec_idx_remove_300)} instances')
len(x_sec)
# Method 2 = By balancing the data
xsecc,ysecc = class_div(x_sec,ysec_ie,8228,5)
print('*'*40)
xcitt,ycitt = class_div(x_cit,ycit_ie,8228,2)
pd.Series(ycitt).value_counts()
#for v-1 model = method - 1 = without balancing the data
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),citm))

y = torch.tensor(y[:-2])
x = torch.tensor(x[:-2])
print('x.shape : ',x.shape)
print('y.shape : ',y.shape)

xcit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),x_cit))

xsec = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),x_sec))

sec = [(x,y) for x,y in zip(xsec,ysec_ie)]
sec = random.sample(sec, 8232)

cit = [(x,y) for x,y in zip(xcit,ycit_ie)]
cit = random.sample(cit, 8232)

xcit = torch.tensor([t[0] for t in cit])
ycit = torch.tensor([t[1] for t in cit])
xsec = torch.tensor([t[0] for t in sec])
ysec = torch.tensor([t[1] for t in sec])
print(xcit.shape)
print(xsec.shape)
#for v-1 model = method - 2 = after balancing the data
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),citm))

y = torch.tensor(y)
x = torch.tensor(x)
print('x.shape : ',x.shape)
print('y.shape : ',y.shape)

xcit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),xcitt))
xcit = torch.tensor(xcit)
ycit = torch.tensor(ycitt)
print('xcit shape : ',xcit.shape)

xsec = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),xsecc))
xsec = torch.tensor(xsec)
ysec = torch.tensor(ysecc)
print('xsec shape : ',xsec.shape)
print(ysec.shape)
print(ycit.shape)
pd.Series(ysec).value_counts()
# for v-2 model : including all 3 scaffolds
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),citm))

m = [(x,y) for x,y in zip(x,y)]
m = random.sample(m, 3000)

x = [i[0] for i in m]
y = [i[1] for i in m]

y = torch.tensor(y)
x = torch.tensor(x)
print(x.shape)
print(y.shape)

xcit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),x_cit))
xsec = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),x_sec))

sec = [(x,y) for x,y in zip(xsec,ysec_ie)]
sec = random.sample(sec, 3000)

cit = [(x,y) for x,y in zip(xcit,ycit_ie)]
cit = random.sample(cit, 3000)

xcit = torch.tensor([t[0] for t in cit])
ycit = torch.tensor([t[1] for t in cit])
xsec = torch.tensor([t[0] for t in sec])
ysec = torch.tensor([t[1] for t in sec])
print(xcit.shape)
print(xsec.shape)<jupyter_output><empty_output><jupyter_text>**ACL Data prep**<jupyter_code>acl_x_cit = acl_citdata['cleaned_cite_text'].values.tolist()
len(acl_x_cit)
acl_ycit = acl_citdata['is_citation'].values.tolist()
acl_ycit_ie = list(map(lambda x : int(x),acl_ycit))
len(acl_ycit_ie)
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
acl_xcit_idx_remove_300 = remove_idx(acl_x_cit,300)
for i in sorted(acl_xcit_idx_remove_300, reverse = True):  
    del acl_x_cit[i]
    del acl_ycit_ie[i]
print(f'removed {len(acl_xcit_idx_remove_300)} instances')
acl_x_sec = acl_secdata['text'].values.tolist()
len(acl_x_sec)
acl_y_sec = acl_secdata['section_name'].values.tolist()
labels={'introduction':0,'related work':1,'method':2,'experiments':3,'conclusion':4}
acl_ysec_ie = list(map(lambda t : labels[t],acl_y_sec))
len(acl_ysec_ie)
acl_xsec_idx_remove_300 = remove_idx(acl_x_sec,300)
for i in sorted(acl_xsec_idx_remove_300, reverse = True):  
    del acl_x_sec[i]
    del acl_ysec_ie[i]
print(f'removed {len(acl_xsec_idx_remove_300)} instances')
#for v-1 model = method - 1 = without balancing the main data
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
acl_tr = list(acl_pdm['cleaned_cite_text'])
acl_y = acl_pdm['intent'].values.tolist()
acl_label={'Background':0,'CompareOrContrast':1,'Extends':2,'Future':3,'Motivation':4,'Uses':5}
acl_y = list(map(lambda t : acl_label[t],acl_y))

print(acl_y[:5])

acl_tr_idx_remove_300 = remove_idx(acl_tr,300)
for i in sorted(acl_tr_idx_remove_300, reverse = True):  
    del acl_tr[i]
    del acl_y[i]
print(f'removed {len(acl_tr_idx_remove_300)} instances')

tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
acl_x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),acl_tr))
acl_y = torch.tensor(acl_y)
acl_x = torch.tensor(acl_x)
print(acl_x[:5])
print(acl_x.shape)
print(acl_y[:5])
print(acl_y.shape)

# By balancing the scaffold data only........
xsecc,ysecc = class_div(acl_x_sec,acl_ysec_ie,1688,5)
print('*'*40)
xcitt,ycitt = class_div(acl_x_cit,acl_ycit_ie,1688,2)

xcit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),xcitt))

xsec = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),xsecc))

xsec = torch.tensor(xsec)
xcit = torch.tensor(xcit)
ycit = torch.tensor(ycitt)
ysec = torch.tensor(ysecc)
print(xcit.shape)
print(ycit.shape)
print(xsec.shape)
print(ysec.shape)
# validation data
vm=[]
with jsonlines.open(acl_val_main) as f:
    for line in f.iter():
        vm.append(line)
acl_pdvm = pd.DataFrame(vm)
acl_pdvm        

acl_citvm=acl_pdvm['text'].values.tolist()
acl_yc = acl_pdvm['intent'].values.tolist()
label={'Background':0,'CompareOrContrast':1,'Extends':2,'Future':3,'Motivation':4,'Uses':5}
acl_vy = list(map(lambda t : label[t],acl_yc))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
vx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),acl_citvm))
vy = torch.tensor(acl_vy[:-2])
vx = torch.tensor(vx[:-2])
print(vx.shape)
acl_pdtm.info()
# test data
testm=[]
with jsonlines.open(acl_test_main) as f:
    for line in f.iter():
        testm.append(line)
acl_pdtm = pd.DataFrame(testm)
acl_pdtm        

acl_cittem=acl_pdtm[:-1]['text'].values.tolist()
acl_yc = acl_pdtm[:-1]['intent'].values.tolist()
label={'Background':0,'CompareOrContrast':1,'Extends':2,'Future':3,'Motivation':4,'Uses':5}
ty = list(map(lambda t : label[t],acl_yc))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),acl_cittem))
ty = torch.tensor(ty[:-2])
tx = torch.tensor(tx[:-2])
print(tx.shape)<jupyter_output><empty_output><jupyter_text>check for torch.mm used in attention class (rough) when w for att. weights shape = (2*hidden,1)<jupyter_code>wl=torch.rand(10,1)
wl
xl=torch.randn(3,10,10)
n = torch.mm(xl.view(-1,10),wl)
n.shape
xl
xl[1][0]
n.view(-1,10,1)[0]
F.softmax(n.view(-1,10,1),dim=1)[1]
# custom softmax function for cross-checking
s=0
l=[]
for i in range(n.view(-1,10,1)[1].shape[0]):
    t = n.view(-1,10,1)[1][i]
    l.append(np.exp(t.item()))
    s+=np.exp(t.item())
l/s<jupyter_output><empty_output><jupyter_text>check for torch.bmm used in attention class (rough) when w in att. weights shape = (batch_size,2*hidden,1)<jupyter_code>x1=torch.rand(3,10,10)
w1=torch.rand(3,10,1)
e1=torch.bmm(x1,w1)
e1
e1[0]
# custom softmax to crosscheck the values 
s1=0
l1=[]
for i in range(e1.shape[1]):
    t = e1[1][i]
    l1.append(np.exp(t.item()))
    s1+=np.exp(t.item())
l1/s1
F.softmax(e1,dim=1)<jupyter_output><empty_output><jupyter_text>**MODEL v-1**<jupyter_code>device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
class attention(nn.Module):
    def __init__(self,batch_size,hidden_size=50,max_len=300):
        super().__init__()
        k = torch.empty(batch_size,2*hidden_size,1)
        self.w = torch.nn.init.xavier_uniform_(k).to(device)
        self.hidden = hidden_size
        self.max_len = max_len
    def forward(self,x):
        #e = torch.mm(x.view(-1,2*hidden_size),self.w).view(-1,max_len,1)
        # x.shape = (batch_size,max_len,2*hidden_size), w.shape = (2*hidden_size,1)
        # e.shape = (batch_size,max_len,1)
        # later on also try with w.shape = (batch_size,2*hidden,1) ; use torch.bmm(x,w) in that case instead of above np.dot implementation
        e = torch.bmm(x,self.w)/math.sqrt(self.hidden)
        att_weights = F.softmax(e,dim=1) # attention weights shape = (batch_size,max_len,1)
        out = torch.bmm(x.transpose(1,2),att_weights)
        return out
#feed forward of main task
class feedforward1(nn.Module):
    def __init__(self,data):
        super().__init__()
        
        n = 3 if data=='sci' else 6
            
        self.drop = nn.Dropout(p=0.2)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,n)
    def forward(self,x):
        x = self.drop(x)
        lin_out = self.lin(x)
        x = self.relu(lin_out)
        x = self.out(x)
        return (x,lin_out) 
#feed forward of section scaffold
class feedforward2(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.2)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,5)
    def forward(self,x):
        x = self.drop(x)
        lin_out = self.lin(x)
        x = self.relu(lin_out)
        x = self.out(x)
        return (x,lin_out) 
#feed forward of citation worthiness scaffold
class feedforward3(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.2)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,2)
    def forward(self,x):
        x = self.drop(x)
        lin_out = self.lin(x)
        x = self.relu(lin_out)
        x = self.out(x)
        return (x,lin_out) 
class model(nn.Module):
    def __init__(self,batch_size):
        super().__init__()
        self.batch_size = batch_size
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
        self.att = attention(batch_size)
        self.main_sci = feedforward1(data='sci')
        self.main_pk = feedforward1(data='pk')
        self.sec = feedforward2()
        self.cit = feedforward3()
    def forward(self,x,n,data=None):
        xbert=self.BertModel(x)
        lstm,_=self.lstm(xbert[0])
        att = self.att(lstm).view(self.batch_size,100)
        if(n==1 and data=='sci'):
            return self.main_sci(att)[0]
        
        elif(n==1 and data=='pk'):
            return self.main_pk(att)[0]
        
        elif(n==2):
            return self.sec(att)[0]
        
        elif(n==3):
            return self.cit(att)[0]
        else:
            # predicting == training done!!
            if(data=='sci'):
                z,last = self.main_sci(att)
                _,last_sec = self.sec(att)
                _,last_cit = self.cit(att)
            else:
                z,last = self.main_pk(att)
                _,last_sec = self.sec(att)
                _,last_cit = self.cit(att)
            z = F.softmax(z,dim=1)
            z = torch.argmax(z,dim=1)
            return (z,last,att,last_cit,last_sec)
        
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=100),list(pdm['string'])))
x = torch.tensor(x)
luv = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")(x[0].view(1,-1))
luv[0].shape
lst = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)(luv[0])
lst[0].shape
att = attention(1)(lst[0].cuda())
att.shape<jupyter_output><empty_output><jupyter_text>**MODEL v-2**<jupyter_code>device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
class attention(nn.Module):
    def __init__(self,batch_size,hidden_size=50,max_len=70):
        super().__init__()
        k = torch.empty(batch_size,2*hidden_size,1)
        self.w = torch.nn.init.xavier_uniform_(k).to(device)
        self.hidden = hidden_size
        self.max_len = max_len
    def forward(self,x):
        #e = torch.mm(x.view(-1,2*hidden_size),self.w).view(-1,max_len,1)
        # x.shape = (batch_size,max_len,2*hidden_size), w.shape = (2*hidden_size,1)
        # e.shape = (batch_size,max_len,1)
        # later on also try with w.shape = (batch_size,2*hidden,1) ; use torch.bmm(x,w) in that case instead of above np.dot implementation
        e = torch.bmm(x,self.w)/math.sqrt(self.hidden)
        att_weights = F.softmax(e,dim=1) # attention weights shape = (batch_size,max_len,1)
        out = torch.bmm(x.transpose(1,2),att_weights)
        return out
#feed forward of main task
class feedforward1(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,3)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
#feed forward of section scaffold
class feedforward2(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,5)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
#feed forward of citation worthiness scaffold
class feedforward3(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,2)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
#feed forward of cited paper title scaffold
class feedforward4(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,3)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
class modelv2(nn.Module):
    def __init__(self,batch_size):
        super().__init__()
        self.batch_size = batch_size
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
        self.att = attention(batch_size)
        self.main = feedforward1()
        self.sec = feedforward2()
        self.cit = feedforward3()
        self.tit = feedforward4()
        
    def forward(self,x,n):
        xbert=self.BertModel(x)
        lstm,_=self.lstm(xbert[0])
        z = self.att(lstm).view(self.batch_size,100)
        if(n==1):
            return self.main(z)
        
        elif(n==2):
            return self.sec(z)
        
        elif(n==3):
            return self.cit(z)
        elif(n==4):
            return self.tit(z)
        else:
            #predicting == training done!!
            z = self.main(z)
            z = F.softmax(z,dim=1)
            return torch.argmax(z,dim=1)<jupyter_output><empty_output><jupyter_text>**Model v-3**<jupyter_code>device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
k = torch.empty(1,4,1)
hidden = 50
print(k)
we = torch.nn.init.xavier_uniform_(k)
wea = nn.Parameter(we)
print(wea)
wa = wea
for i in range(4-1):
    wa = torch.cat((wa,wea),0)
x = torch.rand(4,20,4)
#wa = nn.Parameter(wa)
e = torch.bmm(x,wa)/math.sqrt(hidden)
print(wa)
print('*'*40)
print(e)
e.grad_fn
xr = torch.rand(4,300,1)
print(xr)
print(xr.transpose(1,2))
mod.att.we
class attention(nn.Module):
    def __init__(self,batch_size,hidden_size=50,max_len=300):
        super().__init__()
        k = torch.zeros(1,2*hidden_size,1)
        torch.nn.init.kaiming_uniform_(k).to(device)
        self.we = nn.Parameter(k).to(device)
        w = self.we
        for i in range(batch_size-1):
            w = torch.cat((w,self.we),0)
        self.w = w.to(device)
        self.hidden = hidden_size
        self.max_len = max_len
        self.e=0
        self.att_weights=0
        self.out=0
    def forward(self,x):
        #e = torch.mm(x.view(-1,2*hidden_size),self.w).view(-1,max_len,1)
        # x.shape = (batch_size,max_len,2*hidden_size), w.shape = (2*hidden_size,1)
        # e.shape = (batch_size,max_len,1)
        # later on also try with w.shape = (batch_size,2*hidden,1) ; use torch.bmm(x,w) in that case instead of above np.dot implementation
        self.e = torch.bmm(x,self.w)/math.sqrt(self.hidden)
        self.tanh = torch.tanh(self.e)
        self.att_weights = F.softmax(self.tanh,dim=1) # attention weights shape = (batch_size,max_len,1)
        self.out = torch.bmm(self.att_weights.transpose(1,2),x)
        return self.out
a = attention(100,300)
x = torch.rand(4,300,100)
a(x)
w = torch.zeros(100,1)
nn.init.kaiming_uniform_(w)
we = nn.Parameter(w)
print(we)
b = nn.Parameter(torch.ones(300))
print(b)
xfl = torch.rand(4,300,100)
print(xfl)
print(xfl.contiguous().view(-1, 100))
e = torch.mm(
            xfl.contiguous().view(-1, 100), 
            we
        )
print(e)
print(e.shape)
e = e.view(-1, 300)
print(e)
e = e+b
print(e)
th = torch.tanh(e)
print(th)
a = torch.exp(th)
print(a)
print('a shape : ',a.shape)
print(torch.sum(a, 1, keepdim=True))
a = a/torch.sum(a, 1, keepdim=True)
print(a)
print(a.shape)
xfl.shape
print(torch.unsqueeze(a, -1).shape)
print(xfl * torch.unsqueeze(a, -1).shape)
class attention(nn.Module):
    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
        super(attention, self).__init__(**kwargs)
        
        self.supports_masking = True

        self.bias = bias
        self.feature_dim = feature_dim
        self.step_dim = step_dim
        self.features_dim = 0
        self.a = 0
        self.th = 0
        self.eij = 0
        
        weight = torch.zeros(feature_dim, 1)
        nn.init.kaiming_uniform_(weight)
        self.weight = nn.Parameter(weight)
        print('weight : ',self.weight)
        print('weight shape : ',weight.shape)
        
        if bias:
            self.b = nn.Parameter(torch.zeros(step_dim))
        print('bias shape : ',self.b.shape)
        
    def forward(self, x, mask=None):
        feature_dim = self.feature_dim 
        step_dim = self.step_dim
#         print('x shape : ',x.shape)

        self.eij = torch.mm(
            x.contiguous().view(-1, feature_dim), 
            self.weight
        ).view(-1, step_dim)
        
#         print('eij shape : ',self.eij.shape)
        
        if self.bias:
            self.eij = self.eij + self.b
            
        self.th = torch.tanh(self.eij)
#         print('tanh out shape : ',self.th.shape)
        a = torch.exp(self.th)
#         print('a shape : ',a.shape)
        
        if mask is not None:
            a = a * mask

        self.a = a / (torch.sum(a, 1, keepdim=True) + 1e-10)
#         print('a divided by sum shape : ',self.a.shape)

        weighted_input = x * torch.unsqueeze(self.a, -1)
#         print('weighted input : ',weighted_input.shape)
        return torch.sum(weighted_input, 1)
a = torch.rand(4,300)
a = torch.unsqueeze(a,-1)
e = x*a
torch.sum(e,1).shape
#feed forward of main task
class feedforward1(nn.Module):
    def __init__(self,data):
        super().__init__()
        
        n = 3 if data=='sci' else 6
        drop = 0.2 if data == 'sci' else 0.3
            
        self.drop = nn.Dropout(p=drop)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,n)
    def forward(self,x):
        x = self.drop(x)
        lin_out = self.lin(x)
        x = self.relu(lin_out)
        x = self.out(x)
        return (x,lin_out) 
#feed forward of section scaffold
class feedforward2(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,5)
    def forward(self,x):
        x = self.drop(x)
        lin_out = self.lin(x)
        x = self.relu(lin_out)
        x = self.out(x)
        return (x,lin_out) 
#feed forward of citation worthiness scaffold
class feedforward3(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,2)
    def forward(self,x):
        x = self.drop(x)
        lin_out = self.lin(x)
        x = self.relu(lin_out)
        x = self.out(x)
        return (x,lin_out) 
#feed forward of citance + cited title scaffold
class feedforward4(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,20)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(20,6)
    def forward(self,x):
        x = self.drop(x)
        lin_out = self.lin(x)
        x = self.relu(lin_out)
        x = self.out(x)
        return (x,lin_out) 
class modelv3(nn.Module):
    def __init__(self,batch_size):
        super().__init__()
        self.batch_size = batch_size
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
#         self.att = attention(batch_size)
        self.att = attention(100,300)
        self.main_sci = feedforward1(data='sci')
        self.main_pk = feedforward1(data='pk')
        self.sec = feedforward2()
        self.cit = feedforward3()
        self.cited = feedforward4()
    def forward(self,x,n,data=None):
        xbert=self.BertModel(x)
        lstm,_=self.lstm(xbert[0])
        at = self.att(lstm)
        if(n==1 and data=='sci'):
            return self.main_sci(at)[0]
        
        elif(n==1 and data=='pk'):
            return self.main_pk(at)[0]
        
        elif(n==2):
            return self.sec(at)[0]
        
        elif(n==3):
            return self.cit(at)[0]
        elif(n==4):
            return self.cited(at)[0]
        else:
            # predicting == training done!!
            if(data=='sci'):
                z,last = self.main_sci(at)
                _,last_sec = self.sec(at)
                _,last_cit = self.cit(at)
                z = F.softmax(z,dim=1)
                z = torch.argmax(z,dim=1) 
                return (z,last,at,last_cit,last_sec,lstm)
            else:
                z,last = self.main_pk(at)
                _,last_sec = self.sec(at)
                _,last_cit = self.cit(at)
                _,last_cited = self.cited(at)
                z = F.softmax(z,dim=1)
                z = torch.argmax(z,dim=1)
                return (z,last,at,last_cit,last_sec,last_cited,lstm)
        
modelv3(4)<jupyter_output><empty_output><jupyter_text>**DATA INPUT**<jupyter_code># validation data
vm=[]
with jsonlines.open(val_main) as f:
    for line in f.iter():
        vm.append(line)
pdvm = pd.DataFrame(vm)
pdvm        
citvm=pdvm['string'].values.tolist()
citvm = [fun(i) for i in citvm]
pdvm['processed_text'] = citvm
yc = pdvm['label'].values.tolist()
label={'background':0,'method':1,'result':2}
vy = list(map(lambda t : label[t],yc))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
vx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),citvm))
vy = torch.tensor(vy)
vx = torch.tensor(vx)
vy.shape
# test data
testm=[]
with jsonlines.open(test_main) as f:
    for line in f.iter():
        testm.append(line)
pdtm = pd.DataFrame(testm)    
pdtm.loc[pdtm['string'] == 'Moreover, in our analyses, the antibody responses to vaccination were also analyzed separately and our 12-week follow-up to record the immune response to vaccination was much longer than those reported from previous studies where reduction in immunity can be observed (Kiecolt-Glaser et al. 1996; Glaser et al. 1999; Vedhara et al. 1999).']
cittem=pdtm['string'].values.tolist()
cittem = [fun(i) for i in cittem]
pdtm['processed_text'] = cittem
cittem = cittem[:-1]
yc = pdtm[:-1]['label'].values.tolist()
label={'background':0,'method':1,'result':2}
ty = list(map(lambda t : label[t],yc))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),cittem))
ty = torch.tensor(ty)
tx = torch.tensor(tx)
tx.shape
x.shape
x.shape
for i in val_data:
    print(i[0].shape)
batchSize = 12
num_of_feedforwards = 3
#model = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
main_tr = torch.utils.data.TensorDataset(x, y)
main_val = torch.utils.data.TensorDataset(vx, vy)
main_test = torch.utils.data.TensorDataset(tx, ty)
sec_tr = torch.utils.data.TensorDataset(xsec,ysec)
cit_tr = torch.utils.data.TensorDataset(xcit,ycit)
#tit_tr = torch.utils.data.TensorDataset(x_tit,y_tit)

 
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

train_sampler = torch.utils.data.RandomSampler(sec_tr)
sec_data = torch.utils.data.DataLoader(sec_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

train_sampler = torch.utils.data.RandomSampler(cit_tr)
cit_data = torch.utils.data.DataLoader(cit_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

val_sampler = torch.utils.data.RandomSampler(main_val)
val_data = torch.utils.data.DataLoader(main_val, sampler=val_sampler, batch_size=batchSize//num_of_feedforwards)

test_sampler = torch.utils.data.RandomSampler(main_test)
test_data = torch.utils.data.DataLoader(main_test, sampler=test_sampler, batch_size=batchSize//num_of_feedforwards)

# train_sampler = torch.utils.data.RandomSampler(tit_tr)
# tit_data = torch.utils.data.DataLoader(tit_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)<jupyter_output><empty_output><jupyter_text>**Training**<jupyter_code>mod = torch.load('../input/cohan-finetuned-on-acl/cohan_model_finetuned_acl_without_freezing_ep20.pt')
mod.eval()
print(list(mod.parameters())[0].grad)
for name,param in mod.named_parameters():
    if(name.split('.')[0] == 'main_pk' or name.split('.')[0] == 'cited'):
        print(name)
print(list(mod.named_parameters())[-13][1].grad)
mod = torch.load('../input/cohan-v3-models/cohan_modelv3_ep6.pt')
mod.eval()
def plot_grad_flow(named_parameters):
    ave_grads = []
    layers = []
    for n, p in named_parameters:
        if(p.requires_grad) and ("bias" not in n):
            layers.append(n)
            ave_grads.append(p.grad.abs().mean())
    plt.plot(ave_grads, alpha=0.3, color="b")
    plt.hlines(0, 0, len(ave_grads)+1, linewidth=1, color="k" )
    plt.xticks(range(0,len(ave_grads), 1), layers, rotation="vertical")
    plt.xlim(xmin=0, xmax=len(ave_grads))
    plt.xlabel("Layers")
    plt.ylabel("average gradient")
    plt.title("Gradient flow")
    plt.grid(True)
name
for name,i in mod.named_parameters():
    try:
        print(i.grad.device,name)
    except:
        pass
#training for models
# mod=modelv3(batchSize//num_of_feedforwards)
mod.to(device)

for name,param in mod.named_parameters():
    if(name.split('.')[0] == 'main_pk' or name.split('.')[0] == 'cited'):
        param.requires_grad = False

lambd1 = 0.05   #lambd1 for influence of section scaffold
lambd2 = 0.1    #lambd2 for influence of citation worthiness scaffold
#lambd3 = 0.1    #lambd3 for influence of cited paper title scaffold

loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adadelta(mod.parameters(), lr = 0.01)

loss_list = []
f1_list = []

for epoch in range(3,6):
    running_loss = 0
    ypr=[]
    ytr=[]
    y_eval=[]
    ypr_eval=[]
    
    print(f'gradients before epoch{epoch+1} : ')
    print('main_pk out :')
    print(list(mod.named_parameters())[-13][1].grad)
    print('main_sci out :')
    print(list(mod.named_parameters())[-17][1].grad)
    
# training--------------------
    mod.train()
#     print(mod.att.att_weights.requires_grad)
#     print(mod.att.out.requires_grad)
    for i,data in enumerate(zip(train_data,sec_data,cit_data)):
        m = data[0]
        s = data[1]
        c = data[2]
       # t = data[3]
        
        in_main,tar_main = m[0].to(device),m[1].to(device)
        in_sec,tar_sec = s[0].to(device),s[1].to(device)
        in_cit,tar_cit = c[0].to(device),c[1].to(device)
        #in_tit,tar_tit = t[0].to(device),t[1].to(device)
        
        optimizer.zero_grad()
        
        main = mod(in_main,1,'sci')
        sec = mod(in_sec,2)
        cit = mod(in_cit,3)
        #tit = mod(in_cit,4)
        
        loss_main = loss(main,tar_main)
        loss_sec = loss(sec,tar_sec)
        loss_cit = loss(cit,tar_cit)
        #loss_tit = loss(tit,tar_tit)

        overall_loss = (loss_main + lambd1*loss_sec + lambd2*loss_cit)/num_of_feedforwards  # becoz initially the summation is avg loss per mini batch(8) but we need avg loss per mini batch(24)
        overall_loss.backward()
        torch.nn.utils.clip_grad_norm_(mod.parameters(), 5)
#         plot_grad_flow(mod.named_parameters())
        optimizer.step()
        
        running_loss += overall_loss.item()
        if(i%100 == 99):
            loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(running_loss / 100,3)})
            print('[epoch : %d,batch : %5d] loss: %.3f' %(epoch + 1, i + 1, running_loss / 100))
            
#             print('gradients : ')
#             print('main_sci out :')
#             print(list(mod.named_parameters())[-13][1].grad)
#             print('main_pk out :')
#             print(list(mod.named_parameters())[-9][1].grad)
#             print('#'*50)
            running_loss = 0.0
   
# validation ------------------------
    with torch.no_grad():
        mod.eval()
        
        # calculating f1_score for train data
        for d in train_data:
            x = d[0].to(device)
            y = d[1].to(device)
            for yt in y.cpu():
                ytr.append(yt)
            y_pred = mod(x,0,'sci')[0].cpu()
            for yt in y_pred:
                ypr.append(yt)
                
        f1 = f1_score(ytr,ypr,average='macro')
        
        # calculating f1_score for validation data
        for d in val_data:
            xv = d[0].to(device)
            yv = d[1].to(device)
            for yt in yv.cpu():
                y_eval.append(yt)
            y_pred = mod(xv,0,'sci')[0].cpu()
            for yt in y_pred:
                ypr_eval.append(yt)
                
        val_f1 = f1_score(y_eval,ypr_eval,average='macro')
        
        f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})
        
    print('*'*40)
    print('train confusion matrix : ')
    print(confusion_matrix(ytr, ypr))
    print('*'*40)
    print('val confusion matrix : ')
    print(confusion_matrix(y_eval, ypr_eval))
    print('*'*40)
    print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f' %(epoch+1, f1, val_f1))
    print('*'*40)
    torch.save(mod, f'./cohan_modelv3_att_renew_same_w_dropout0.3_ep{epoch+1}.pt')
    print('#'*40)
    print('attention weight vector : ')
    print(mod.att.weight)
    print('attention eij : ')
    print(mod.att.eij)
    print('attention att weights : ')
    print(mod.att.a)
#     if((epoch+1)%2==0):
#         torch.save(mod, f'./cohan_modelv3_ep{epoch+1}.pt')

print('Finished Training!!')  
FileLinks('./')
cittem
tx = tokenizer.encode(cittem[0])
len(tx)
mod = torch.load('./cohan_modelv3_att_renew_same_w_dropout0.3_ep5.pt') ## cohan_modelv3_dropout_ep#.pt are the renewed attention layer models
cittem
cittem=pdtm['string'].values.tolist()[0]
cittem = [fun(cittem)]
# pdtm['processed_text'] = cittem
yc = pdtm['label'].values.tolist()[0]
label={'background':0,'method':1,'result':2}
ty = label[yc]
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),cittem))
ty = torch.tensor(ty).view(1,-1)
tx = torch.tensor(tx)
ty
tx
x = torch.cat((tx,tx,tx,tx),0)
y = torch.cat((ty,ty,ty,ty),0)
x.shape
mod.att.a
mod(xt,0,'sci')
xt = x.cuda()
yt = y.cuda()
lstm = mod(xt,0,'sci')[-1]
print('output : ',mod(xt,0,'sci')[0])
print('lstm : ',lstm.shape)
# wei = mod.att.we
# e = torch.bmm(lstm,wei)
e = mod.att.eij
print(e.shape)
# att_weights = F.softmax(e,dim=1)
att_weights = mod.att.a
print(att_weights.shape)

import seaborn as sns
import matplotlib.pyplot as plt
#sns.color_palette("coolwarm")
aw = att_weights[0].view(-1,1).cpu().detach().numpy()
print(aw.shape)
aw = aw[:35]
fig, ax = plt.subplots(figsize=(20, 1))
sns.heatmap(aw.transpose(1,0),linewidths=.01,cmap="YlGnBu")
aw
att_weights[0]
cittem
#from transformers import AutoTokenizer, AutoModel,convert_ids_to_tokens
tokenizer.convert_ids_to_tokens(xt[3])[34]
print(list(mod.named_parameters())[-9][1].grad)
import os
os.chdir(r'../working')
from IPython.display import FileLink
FileLink(r'./cohan_modelv3_ep4.pt')
FileLink(r'./cohan_modelv3_ep6.pt')
FileLink(r'./cohan_modelv3_ep8.pt')
FileLink(r'./cohan_modelv3_ep10.pt')
print('weights before the start of training : ')
for name, param in mod.named_parameters():
    if(name.split('.')[0]!='BertModel'):
        print(name,param.data)<jupyter_output><empty_output><jupyter_text>**Saving the model**<jupyter_code>mod.state_dict()
torch.save(mod, 'check_cohan_sci.pt')
import os
os.chdir(r'../working')
from IPython.display import FileLink
FileLink(r'./cohan_model_finetuned_acl_without_freezing_ep45.pt')
torch.save({
            'epoch': epoch,
            'model_state_dict': mod.state_dict(),
            'loss': overall_loss,
            }
            , './model_epoch4.tar')
torch.cuda.memory_summary(device=None, abbreviated=False)<jupyter_output><empty_output><jupyter_text>**Test on acl-arc**<jupyter_code>mod = torch.load('./cohan_model_finetuned_acl_without_freezing_ep40.pt')
mod.eval()
batchSize = 12
num_of_feedforwards = 3
main_tr = torch.utils.data.TensorDataset(acl_x, acl_y)
main_test = torch.utils.data.TensorDataset(tx, ty)
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)
test_sampler = torch.utils.data.RandomSampler(main_test)
test_data = torch.utils.data.DataLoader(main_test, sampler=test_sampler, batch_size=batchSize//num_of_feedforwards)
y_test=[]
ypr_test=[]
with torch.no_grad():
    mod.eval()
    for d in test_data:
        xte = d[0].to(device)
        yte = d[1].to(device)
        for yt in yte.cpu():
            y_test.append(yt)
        y_pred = mod(xte,0,'pk')[0].cpu()
        for yt in y_pred:
            ypr_test.append(yt)
                
    test_f1 = f1_score(y_test,ypr_test,average='macro')
    print('test_f1_score : ',test_f1)
y_test=[]
ypr_test=[]
with torch.no_grad():
    mod.eval()
    for d in train_data:
        xte = d[0].to(device)
        yte = d[1].to(device)
        for yt in yte.cpu():
            y_test.append(yt)
        y_pred = mod(xte,0,'pk')[0].cpu()
        for yt in y_pred:
            ypr_test.append(yt)
                
    test_f1 = f1_score(y_test,ypr_test,average='macro')
    print('train_f1_score : ',test_f1)<jupyter_output><empty_output><jupyter_text>**Models Loading for test on pride data**<jupyter_code>mod_cohan_sci = torch.load('../input/cohan-model/full_model_epoch4.pth')
mod_cohan_sci.eval()
model_v2 = torch.load('./full_modelv2_epoch_g4.pth')
model_v2.eval()
model_v3 = torch.load('../input/cohan-v3-models/cohan_modelv3_ep4.pt')
model_v3.eval()
mod = torch.load('./cohan_modelv3_scaffold4_pk_ep18.pt')
mod.eval()
batchSize = 12
num_of_feedforwards = 3
main_test = torch.utils.data.TensorDataset(tx, ty)
test_sampler = torch.utils.data.RandomSampler(main_test)
test_data = torch.utils.data.DataLoader(main_test, sampler=test_sampler, batch_size=batchSize//num_of_feedforwards)
y_test=[]
ypr_test=[]
with torch.no_grad():
    mod.eval()
    for d in test_data:
        xte = d[0].to(device)
        yte = d[1].to(device)
        for yt in yte.cpu():
            y_test.append(yt)
        y_pred = model_v3(xte,0,'sci')[0].cpu()
        for yt in y_pred:
            ypr_test.append(yt)
                
    test_f1 = f1_score(y_test,ypr_test,average='macro')
    print('test_f1_score : ',test_f1)
y_test=[]
ypr_test=[]
with torch.no_grad():
    mod.eval()
    for d in train_data:
        xte = d[0].to(device)
        yte = d[1].to(device)
        for yt in yte.cpu():
            y_test.append(yt)
        y_pred = model_v3(xte,0,'sci')[0].cpu()
        for yt in y_pred:
            ypr_test.append(yt)
                
    test_f1 = f1_score(y_test,ypr_test,average='macro')
    print('test_f1_score : ',test_f1)
y_test1 = [y.item() for y in y_test]
pd.Series(y_test1).value_counts()
ypr_test1 = [y.item() for y in ypr_test]
pd.Series(ypr_test1).value_counts()
pdtr = pd.read_csv('../input/pride-data/train.csv')
pdte = pd.read_csv('../input/pride-data/test.csv')
pdtr
pdtr['citation_class_label'].value_counts()
pdtr.info()
pdte['citation_context'][29]
pdte['cited_title'][29]
grp = pdtr.groupby('citing_author')
for aut,inst in grp:
    print(aut)
    print(inst[['unique_id','citation_context','citation_class_label']])
pdtr.iloc[[619,620]]
pdtr.iloc[619]['citation_context'] #label : 0 = 'BACKGROUND' = so same sentence as in index 620 but diff. classification of citation
pdtr.iloc[620]['citation_context'] #label : 1 = 'COMPARES_CONTRASTS' = so same sentence as in index 619 but diff. classification of citation
len(pdtr['citation_class_label'].values.tolist())
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
len(pdtr['citation_context'].values.tolist())
len(prtr)
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
label={0:0,1:2,2:0,3:0,4:0,5:1}
ty = list(map(lambda t : label[t],yc))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),prtr))
ty = torch.tensor(ty)
tx = torch.tensor(tx)
tx.shape
ty.shape
batchSize = 12
num_of_feedforwards = 3
pride_test = torch.utils.data.TensorDataset(tx, ty)
prtest_sampler = torch.utils.data.RandomSampler(pride_test)
prtest_data = torch.utils.data.DataLoader(pride_test, sampler=prtest_sampler, batch_size=batchSize//num_of_feedforwards)
y=[]
ypredict=[]
with torch.no_grad():
    mod.eval()
    for d in prtest_data:
        xte = d[0].to(device)
        yte = d[1].to(device)
        for yt in yte.cpu():
            y.append(yt)
        y_pred = mod(xte,0,'pk')[0].cpu()
        for yt in y_pred:
            ypredict.append(yt)
                
    test_f1 = f1_score(y,ypredict,average='macro')
    print('pride_f1_score : ',test_f1)
y1 = [y.item() for y in y]
pd.Series(y1).value_counts()
ypredict1 = [y.item() for y in ypredict]
pd.Series(ypredict1).value_counts()
y=[]
ypredict=[]
with torch.no_grad():
    model_c.eval()
    for d in prtest_data:
        xte = d[0].to(device)
        yte = d[1].to(device)
        for yt in yte.cpu():
            y.append(yt)
        y_pred = model_c(xte,0).cpu()
        for yt in y_pred:
            ypredict.append(yt)
                
    test_f1 = f1_score(y,ypredict,average='macro')
    print('pride_f1_score : ',test_f1)
# pride data prep for model v1

citm=pdtr['citation_context'].values.tolist()

yc = pdtr['citation_class_label'].values.tolist()
# label={0:0,1:2,2:1,3:0,4:0,5:1}
# y = list(map(lambda t : label[t],yc))

x_cit = datac['cleaned_cite_text'].values.tolist()

ycit = datac['is_citation'].values.tolist()
ycit_ie = list(map(lambda x : int(x),ycit))

x_sec = data['text'].values.tolist()

y_sec = data['section_name'].values.tolist()
labels={'introduction':0,'related work':1,'method':2,'experiments':3,'conclusion':4}
ysec_ie = list(map(lambda t : labels[t],y_sec))

tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=100),citm))

y = torch.tensor(yc)
x = torch.tensor(x)
vx = x[2600:]
vy = y[2600:]
x=x[:2600]
y=y[:2600]
print(x.shape)
print(y.shape)



xcit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=100),x_cit))
xsec = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=100),x_sec))

sec = [(x,y) for x,y in zip(xsec,ysec_ie)]
sec = random.sample(sec, 2600)

cit = [(x,y) for x,y in zip(xcit,ycit_ie)]
cit = random.sample(cit, 2600)

xcit = torch.tensor([t[0] for t in cit])
ycit = torch.tensor([t[1] for t in cit])
xsec = torch.tensor([t[0] for t in sec])
ysec = torch.tensor([t[1] for t in sec])
print(xsec.shape)
print(xcit.shape)
# pride data prep for model v2
x_tit = pdtr['cited_title'].values.tolist()

yc = pdtr['citation_class_label'].values.tolist()
label={0:0,1:2,2:1,3:0,4:0,5:1}
y_tit = list(map(lambda t : label[t],yc))

tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x_tit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),x_tit))
tit = [(x,y) for x,y in zip(x_tit,y_tit)]
tit = random.sample(tit, 2600)

x_tit = torch.tensor([t[0] for t in tit])
y_tit = torch.tensor([t[1] for t in tit])

print(x_tit.shape)
print(y_tit.shape)


citm=pdtr['citation_context'].values.tolist()

yc = pdtr['citation_class_label'].values.tolist()
label={0:0,1:2,2:1,3:0,4:0,5:1}
y = list(map(lambda t : label[t],yc))

x_cit = datac['cleaned_cite_text'].values.tolist()

ycit = datac['is_citation'].values.tolist()
ycit_ie = list(map(lambda x : int(x),ycit))

x_sec = data['text'].values.tolist()

y_sec = data['section_name'].values.tolist()
labels={'introduction':0,'related work':1,'method':2,'experiments':3,'conclusion':4}
ysec_ie = list(map(lambda t : labels[t],y_sec))

tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),citm))

y = torch.tensor(y)
x = torch.tensor(x)
vx = x[2600:]
vy = y[2600:]
x=x[:2600]
y=y[:2600]
print(x.shape)
print(y.shape)


xcit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),x_cit))
xsec = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=70),x_sec))

sec = [(x,y) for x,y in zip(xsec,ysec_ie)]
sec = random.sample(sec, 2600)

cit = [(x,y) for x,y in zip(xcit,ycit_ie)]
cit = random.sample(cit, 2600)

xcit = torch.tensor([t[0] for t in cit])
ycit = torch.tensor([t[1] for t in cit])
xsec = torch.tensor([t[0] for t in sec])
ysec = torch.tensor([t[1] for t in sec])
print(xsec.shape)
print(xcit.shape)
batchSize = 12
num_of_feedforwards = 3  # 4 for v2 model and 3 for v1 model

#model = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")

main_tr = torch.utils.data.TensorDataset(x, y)
main_val = torch.utils.data.TensorDataset(vx, vy)
sec_tr = torch.utils.data.TensorDataset(xsec,ysec)
cit_tr = torch.utils.data.TensorDataset(xcit,ycit)
#tit_tr = torch.utils.data.TensorDataset(x_tit,y_tit)
 
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

train_sampler = torch.utils.data.RandomSampler(sec_tr)
sec_data = torch.utils.data.DataLoader(sec_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

train_sampler = torch.utils.data.RandomSampler(cit_tr)
cit_data = torch.utils.data.DataLoader(cit_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

val_sampler = torch.utils.data.RandomSampler(main_val)
val_data = torch.utils.data.DataLoader(main_val, sampler=val_sampler, batch_size=batchSize//num_of_feedforwards)

# train_sampler = torch.utils.data.RandomSampler(tit_tr)
# tit_data = torch.utils.data.DataLoader(tit_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)
model_c.parameters()
mod=model(batchSize//3)
model_c.state_dict()
mod.state_dict()
for param in mod.parameters():
    print(type(param), param.size())
model_c.to(device)
#Training of v1 models on pride data

#mod=model(batchSize//3)
mod.to(device)

lambd1 = 0.05   #lambd1 for influence of section scaffold
lambd2 = 0.05    #lambd2 for influence of citation worthiness scaffold

# loss = torch.nn.CrossEntropyLoss()
# optimizer = torch.optim.Adadelta(mod.parameters(), lr = 0.01)

loss_list = []
f1_list = []

for epoch in range(10):
    running_loss = 0
    ypr=[]
    ytr=[]
    y_eval=[]
    ypr_eval=[]
 
# training--------------------
    mod.train()
    for i,data in enumerate(zip(train_data,sec_data,cit_data)):
        m = data[0]
        s = data[1]
        c = data[2]
        
        in_main,tar_main = m[0].to(device),m[1].to(device)
        in_sec,tar_sec = s[0].to(device),s[1].to(device)
        in_cit,tar_cit = c[0].to(device),c[1].to(device)
        
        optimizer.zero_grad()
        
        main = mod(in_main,1,'pk')
        sec = mod(in_sec,2)
        cit = mod(in_cit,3)
        
        loss_main = loss(main,tar_main)
        loss_sec = loss(sec,tar_sec)
        loss_cit = loss(cit,tar_cit)

        overall_loss = (loss_main + lambd1*loss_sec + lambd2*loss_cit)/num_of_feedforwards  # becoz initially the summation is avg loss per mini batch(8) but we need avg loss per mini batch(24)
        overall_loss.backward()
        torch.nn.utils.clip_grad_norm_(mod.parameters(), 5)
        optimizer.step()
        
        running_loss += overall_loss.item()
        if(i%100 == 99):
            loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(running_loss / 100,3)})
            print('[epoch : %d,batch : %5d] loss: %.3f' %(epoch + 1, i + 1, running_loss / 100))
            running_loss = 0.0
   
# validation ------------------------
    with torch.no_grad():
        mod.eval()
        
        # calculating f1_score for train data
        for d in train_data:
            x = d[0].to(device)
            y = d[1].to(device)
            for yt in y.cpu():
                ytr.append(yt)
            y_pred = mod(x,0,'pk').cpu()
            for yt in y_pred:
                ypr.append(yt)
                
        f1 = f1_score(ytr,ypr,average='macro')
        
        # calculating f1_score for validation data
        for d in val_data:
            xv = d[0].to(device)
            yv = d[1].to(device)
            for yt in yv.cpu():
                y_eval.append(yt)
            y_pred = mod(xv,0,'pk').cpu()
            for yt in y_pred:
                ypr_eval.append(yt)
                
        val_f1 = f1_score(y_eval,ypr_eval,average='macro')
        
        f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})
    
    print('*'*40)
    print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f' %(epoch+1, f1, val_f1))
    print('*'*40)

print('Finished Training!!')            
torch.save(mod, './full_model_v1_epoch32_ml100.pt')
mod = torch.load('./cohan_modelv3_scaffold4_pk_ep20.pt')
mod.eval()
### Submission file generation

prte = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdte['citation_context']))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=250),prte))
tx = torch.tensor(tx)

ind = list(pdte['unique_id'])

pred = []
with torch.no_grad():
    mod.eval()
    for i in range(0,len(tx),4):
        l=[]
        x = tx[i:i+4]
        idx = ind[i:i+4]
        x = x.to(device)
        y_pr = mod(x,0,'pk')[0].cpu()
        for j in range(len(x)):
            l = [idx[j],y_pr[j].item()]
            pred.append(l)
        
df = pd.DataFrame(pred, columns = ['unique_id', 'citation_class_label']) 
df.set_index('unique_id', inplace = True)

print(df)
df.to_csv('./submission1.csv') 
p = [i[1] for i in pred]        #ep24 with SMOTE
pd.Series(p).value_counts()
p = [i[1] for i in pred]          #ep20 without SMOTE
pd.Series(p).value_counts()
mod(x,0,'pk')[0].shape
for d in train_data:
    print(d[0].shape)
#Training of v2 models on pride data

#mod=model(batchSize//3)
mod.to(device)

lambd1 = 0.05   #lambd1 for influence of section scaffold
lambd2 = 0.1    #lambd2 for influence of citation worthiness scaffold
lambd3 = 0.1    #lambd3 for influence of cited paper title scaffold

loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adadelta(mod.parameters(), lr = 0.01)

loss_list = []
f1_list = []

for epoch in range(2):
    running_loss = 0
    ypr=[]
    ytr=[]
    y_eval=[]
    ypr_eval=[]
 
# training--------------------
    mod.train()
    for i,data in enumerate(zip(train_data,sec_data,cit_data,tit_data)):
        m = data[0]
        s = data[1]
        c = data[2]
        t = data[3]
        
        in_main,tar_main = m[0].to(device),m[1].to(device)
        in_sec,tar_sec = s[0].to(device),s[1].to(device)
        in_cit,tar_cit = c[0].to(device),c[1].to(device)
        in_tit,tar_tit = t[0].to(device),t[1].to(device)
        
        optimizer.zero_grad()
        
        main = mod(in_main,1)
        sec = mod(in_sec,2)
        cit = mod(in_cit,3)
        tit = mod(in_tit,4)
        
        loss_main = loss(main,tar_main)
        loss_sec = loss(sec,tar_sec)
        loss_cit = loss(cit,tar_cit)
        loss_tit = loss(tit,tar_tit)

        overall_loss = (loss_main + lambd1*loss_sec + lambd2*loss_cit + lambd3*loss_tit)/num_of_feedforwards  # becoz initially the summation is avg loss per mini batch(8) but we need avg loss per mini batch(24)
        overall_loss.backward()
        torch.nn.utils.clip_grad_norm_(mod.parameters(), 5)
        optimizer.step()
        
        running_loss += overall_loss.item()
        if(i%100 == 99):
            loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(running_loss / 100,3)})
            print('[epoch : %d,batch : %5d] loss: %.3f' %(epoch + 1, i + 1, running_loss / 100))
            running_loss = 0.0
   
# validation ------------------------
    with torch.no_grad():
        mod.eval()
        
        # calculating f1_score for train data
        for d in train_data:
            x = d[0].to(device)
            y = d[1].to(device)
            for yt in y.cpu():
                ytr.append(yt)
            y_pred = mod(x,0).cpu()
            for yt in y_pred:
                ypr.append(yt)
                
        f1 = f1_score(ytr,ypr,average='macro')
        
        # calculating f1_score for validation data
        for d in val_data:
            xv = d[0].to(device)
            yv = d[1].to(device)
            for yt in yv.cpu():
                y_eval.append(yt)
            y_pred = mod(xv,0).cpu()
            for yt in y_pred:
                ypr_eval.append(yt)
                
        val_f1 = f1_score(y_eval,ypr_eval,average='macro')
        
        f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})
    
    print('*'*40)
    print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f' %(epoch+1, f1, val_f1))
    print('*'*40)

print('Finished Training!!')            
torch.save(model_c, './full_model_v1_pk_ep8.pt')
os.listdir('./')<jupyter_output><empty_output><jupyter_text>**4 scaffolds model - (for pk and acl)**<jupyter_code>pdtr = pd.read_csv('../input/pride-data/train.csv')
pdte = pd.read_csv('../input/pride-data/test.csv')
pdtr
# code for combining citance and cited title to feed to simp_model

prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t),prtr))
x_cited = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdtr['cited_title'],x)))
x1 = [x+y for x,y in zip(x,x_cited)]
y = torch.tensor(yc)
x1 = torch.tensor(x1)
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=250),prtr))
x = torch.tensor(x)
vx = x[2600:]
vy = y[2600:]
x=x[:2600]
x1 = x1[:2600]
y=y[:2600]
print(x.shape)
print(vx.shape)
print(x1.shape)
pd.Series(y).value_counts()
# only oversampling of minority classes done for now!!!
sampling_strategy = {0:1447,1:600,2:600,3:600,4:600,5:600}  # use this if u wnat to undersample using RandomUnderSampler after oversampling by SMOTE
tr_over = SMOTE(sampling_strategy=sampling_strategy)
tr_under = RandomUnderSampler(sampling_strategy={0:600,1:600,2:600,3:600,4:600,5:600})
x_smote,y_smote = tr_over.fit_sample(x,y)
x_smote,y_smote = tr_under.fit_sample(x_smote,y_smote)

x1_smote,y_smote = tr_over.fit_sample(x1,y)
x1_smote,y_smote = tr_under.fit_sample(x1_smote,y_smote)
print(Counter(y_smote))

sampling_strategy = {0:201,1:100,2:100,3:100,4:100,5:100} # use this if u want to undersample as well after oversampling by SMOTE is done
# while using above dict for sampling strategy of SMOTE, put k_neighbors = 4
val_over = SMOTE(sampling_strategy=sampling_strategy,k_neighbors = 3)
val_under = RandomUnderSampler(sampling_strategy={0:100,1:100,2:100,3:100,4:100,5:100})
vx_smote,vy_smote = val_over.fit_sample(vx,vy)
vx_smote,vy_smote = val_under.fit_sample(vx_smote,vy_smote)
print(Counter(vy_smote))

x_smote = torch.tensor(x_smote)
y_smote = torch.tensor(y_smote)
vx_smote = torch.tensor(vx_smote)
vy_smote = torch.tensor(vy_smote)
x1_smote = torch.tensor(x1_smote)
# vx_smote = vx_smote[:-2]
# vy_smote = vy_smote[:-2]
# x_smote = x_smote[:-2]
# x1_smote = x1_smote[:-2]
# y_smote = y_smote[:-2]
vx_smote.shape
pd.Series(y).value_counts()
batchSize = 8
num_of_feedforwards = 2  # 4 for v2 model and 3 for v1 model

#model = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")

main_tr = torch.utils.data.TensorDataset(x_smote, y_smote)
main_val = torch.utils.data.TensorDataset(vx_smote, vy_smote)
# sec_tr = torch.utils.data.TensorDataset(xsec,ysec)
# cit_tr = torch.utils.data.TensorDataset(xcit,ycit)
tit_tr = torch.utils.data.TensorDataset(x1_smote,y_smote)
 
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

# train_sampler = torch.utils.data.RandomSampler(sec_tr)
# sec_data = torch.utils.data.DataLoader(sec_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

# train_sampler = torch.utils.data.RandomSampler(cit_tr)
# cit_data = torch.utils.data.DataLoader(cit_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

val_sampler = torch.utils.data.RandomSampler(main_val)
val_data = torch.utils.data.DataLoader(main_val, sampler=val_sampler, batch_size=batchSize//num_of_feedforwards)

train_sampler = torch.utils.data.RandomSampler(tit_tr)
tit_data = torch.utils.data.DataLoader(tit_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)
mod = torch.load('../input/cohan-v3-models/cohan_modelv3_ep4.pt')
mod.eval()
mod.named_parameters
#training for models

# mod.to(device)

for name,param in mod.named_parameters():
    if(name.split('.')[0] == 'main_sci'):
        param.requires_grad = False

lambd1 = 0.05   #lambd1 for influence of section scaffold
lambd2 = 0.1    #lambd2 for influence of citation worthiness scaffold
lambd3 = 0.1    #lambd3 for influence of cited paper title scaffold

# loss = torch.nn.CrossEntropyLoss()
# optimizer = torch.optim.Adadelta(mod.parameters(), lr = 0.01)

loss_list = []
f1_list = []

for epoch in range(10):
    running_loss = 0
    ypr=[]
    ytr=[]
    y_eval=[]
    ypr_eval=[]
    
    print(f'gradients before epoch{epoch+1} : ')
    print('main_sci out :')
    print(list(mod.named_parameters())[-17][1].grad)
    print('main_pk out :')
    print(list(mod.named_parameters())[-13][1].grad)
    
# training--------------------
    mod.train()
    for i,data in enumerate(zip(train_data,tit_data)):
        m = data[0]
        t = data[1]
        
        in_main,tar_main = m[0].to(device),m[1].to(device)
        in_tit,tar_tit = t[0].to(device),t[1].to(device)
        
        optimizer.zero_grad()
        
        main = mod(in_main,1,'pk')
        tit = mod(in_tit,4)
        
        loss_main = loss(main,tar_main)
        loss_tit = loss(tit,tar_tit)

        overall_loss = (loss_main + lambd3*loss_tit)/num_of_feedforwards  # becoz initially the summation is avg loss per mini batch(8) but we need avg loss per mini batch(24)
        overall_loss.backward()
        torch.nn.utils.clip_grad_norm_(mod.parameters(), 5)
        optimizer.step()
        
        running_loss += overall_loss.item()
        if(i%100 == 99):
            loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(running_loss / 100,3)})
            print('[epoch : %d,batch : %5d] loss: %.3f' %(epoch + 1, i + 1, running_loss / 100))
            
#             print('gradients : ')
#             print('main_sci out :')
#             print(list(mod.named_parameters())[-13][1].grad)
#             print('main_pk out :')
#             print(list(mod.named_parameters())[-9][1].grad)
#             print('#'*50)
            running_loss = 0.0
   
# validation ------------------------
    with torch.no_grad():
        mod.eval()
        
        # calculating f1_score for train data
        for d in train_data:
            x = d[0].to(device)
            y = d[1].to(device)
            for yt in y.cpu():
                ytr.append(yt)
            y_pred = mod(x,0,'pk')[0].cpu()
            for yt in y_pred:
                ypr.append(yt)
                
        f1 = f1_score(ytr,ypr,average='macro')
        
        # calculating f1_score for validation data
        for d in val_data:
            xv = d[0].to(device)
            yv = d[1].to(device)
            for yt in yv.cpu():
                y_eval.append(yt)
            y_pred = mod(xv,0,'pk')[0].cpu()
            for yt in y_pred:
                ypr_eval.append(yt)
                
        val_f1 = f1_score(y_eval,ypr_eval,average='macro')
        
        f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})
        
    print('*'*40)
    print('train confusion matrix : ')
    print(confusion_matrix(ytr, ypr))
    print('*'*40)
    print('val confusion matrix : ')
    print(confusion_matrix(y_eval, ypr_eval))
    print('*'*40)
    print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f' %(epoch+1, f1, val_f1))
    print('*'*40)
    if((epoch+1)%2==0):
        torch.save(mod, f'./cohan_modelv3_scaffold4_pk_smote_ep{epoch+21}.pt')

print('Finished Training!!')            
import os
from IPython.display import FileLinks
os.chdir(r'../working')
FileLinks('./')
# for i in os.listdir('./'):
#     if(i[:5]=='cohan'):
#         FileLink(os.path.join(r'./',i))
#         break
i<jupyter_output><empty_output><jupyter_text>**Exp - 0 Pride_Knoth Simple classifier**<jupyter_code>pdtr['cited_title']
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t),prtr))
x_cited = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdtr['cited_title'],x)))
x = [x+y for x,y in zip(x,x_cited)]
y = torch.tensor(yc)
x = torch.tensor(x)
prtr
x.shape
vx = x[2600:]
vy = y[2600:]
x=x[:2600]
y=y[:2600]
print(x.shape)
print(vx.shape)
pd.Series(y).value_counts()
tr_over = SMOTE(sampling_strategy={0:1447,1:600,2:600,3:600,4:600,5:600})
tr_under = RandomUnderSampler(sampling_strategy={0:600,1:600,2:600,3:600,4:600,5:600})
x_smote,y_smote = tr_over.fit_sample(x,y)
x_smote,y_smote = tr_under.fit_sample(x_smote,y_smote)
print(Counter(y_smote))

val_over = SMOTE(sampling_strategy={0:201,1:100,2:100,3:100,4:100,5:100},k_neighbors=3)
val_under = RandomUnderSampler(sampling_strategy={0:100,1:100,2:100,3:100,4:100,5:100})
vx_smote,vy_smote = val_over.fit_sample(vx,vy)
vx_smote,vy_smote = val_under.fit_sample(vx_smote,vy_smote)
print(Counter(vy_smote))

x_smote = torch.tensor(x_smote)
y_smote = torch.tensor(y_smote)
vx_smote = torch.tensor(vx_smote)
vy_smote = torch.tensor(vy_smote)
batchsize = 4
main_tr = torch.utils.data.TensorDataset(x_smote, y_smote)
main_val = torch.utils.data.TensorDataset(vx_smote, vy_smote)
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchsize)
val_sampler = torch.utils.data.RandomSampler(main_val)
val_data = torch.utils.data.DataLoader(main_val, sampler=val_sampler, batch_size=batchsize)
x[0].shape
t[1].shape
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
class simple_model(nn.Module):
    def __init__(self,batch_size):
        super().__init__()
        self.batch_size = batch_size
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
        self.att = attention(batch_size)
        self.lin1 = nn.Linear(100,32)
        self.relu = torch.nn.ReLU()
        self.drop = nn.Dropout(p=0.3)
#         self.lin2 = nn.Linear(64,32)
        self.lin2 = nn.Linear(32,6)
        
    def forward(self,x):
        xbert=self.BertModel(x)
        lstm,_=self.lstm(xbert[0])
        att_layer = self.att(lstm).view(self.batch_size,100)
        x = self.drop(att_layer)
        last_lin = self.lin1(x)
        x = self.relu(last_lin) 
        x = self.lin2(x)
        return (x,att_layer,last_lin)
pred.shape
model(inp)
for i,t in enumerate(train_data):
    inp,out = t[0].to(device),t[1].to(device)
    print(inp.shape)
model = simple_model(batchsize)
model.to(device)
loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adadelta(model.parameters(), lr = 0.01)
loss_list = []
f1_list = []

for epoch in range(5):
    total_loss = 0
    ypr=[]
    ytr=[]
    y_eval=[]
    ypr_eval=[]
    
    model.train()
    
    for i,t in enumerate(train_data):
        inp,out = t[0].to(device),t[1].to(device)
        optimizer.zero_grad()
        pred = model(inp)[0]
        loss_batch = loss(pred,out)
        loss_batch.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
        optimizer.step()
        total_loss += (loss_batch*batchsize).item()
        
        if(i%100 == 99):
            loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(total_loss / 100,3)})
            print('[epoch : %d,batch : %5d] loss: %.3f' %(epoch + 1, i + 1, total_loss / 100))
            total_loss = 0.0
            
    with torch.no_grad():
        model.eval()
        
        # calculating f1_score for train data
        for d in train_data:
            x = d[0].to(device)
            y = d[1].to(device)
            for yt in y.cpu():
                ytr.append(yt)
                
            z = model(x)[0]
            z = F.softmax(z,dim=1)
            z = torch.argmax(z,dim=1)    
            y_pred = z.cpu()
            for yt in y_pred:
                ypr.append(yt)
                
        f1 = f1_score(ytr,ypr,average='macro')
        
        # calculating f1_score for validation data
        for d in val_data:
            xv = d[0].to(device)
            yv = d[1].to(device)
            for yt in yv.cpu():
                y_eval.append(yt)
            
            z = model(xv)[0]
            z = F.softmax(z,dim=1)
            z = torch.argmax(z,dim=1)    
            y_pred = z.cpu()
            for yt in y_pred:
                ypr_eval.append(yt)
                
        val_f1 = f1_score(y_eval,ypr_eval,average='macro')  
        f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})
    
    target_names = ['class 0', 'class 1', 'class 2', 'class 3', 'class 4', 'class 5']
    print('*'*40)
    print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f' %(epoch+1, f1, val_f1))
    print('*'*40)
    print('train classification report : ')
    print(classification_report(ytr, ypr, target_names=target_names))
    print('*'*40)
    print('test classification report : ')
    print(classification_report(y_eval,ypr_eval, target_names=target_names))
    print('*'*40)
    if((epoch+1)%10==0):
        torch.save(model, f'./simple_model_ep{epoch+1}.pt')

print('Finished Training!!') 
torch.save(model, './simple_model_ep30_ml250.pt')
### Submission file generation

prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t),prtr))
x_cited = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdtr['cited_title'],x)))
x = [x+y for x,y in zip(x,x_cited)]
y = torch.tensor(yc)
x = torch.tensor(x)

prte = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdte['citation_context']))
# prte = list(map(lambda t : t[0] + ' ' + t[1],zip(prte,pdte['cited_title'])))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t),prte))
tx_cited = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdte['cited_title'],tx)))
tx = [x+y for x,y in zip(tx,tx_cited)]
tx = torch.tensor(tx)
print(tx[0])
ind = list(pdte['unique_id'])

pred = []
with torch.no_grad():
    model.eval()
    for i in range(0,len(tx),4):
        l=[]
        x = tx[i:i+4]
        idx = ind[i:i+4]
        x = x.to(device)
        y_pr = model(x)[0].cpu()
        y_pr = F.softmax(y_pr,dim=1)
        y_pr = torch.argmax(y_pr,dim=1)
        for j in range(len(x)):
            l = [idx[j],y_pr[j].item()]
            pred.append(l)
        
df = pd.DataFrame(pred, columns = ['unique_id', 'citation_class_label']) 
df.set_index('unique_id', inplace = True)

print(df)
df.to_csv('./submission.csv') 
df['citation_class_label'].value_counts()<jupyter_output><empty_output><jupyter_text>**Exp - 1 Multi-Tasking Model**<jupyter_code># citance + cited title data prep
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x1 = list(map(lambda t : tokenizer.encode(t),prtr))
x1_cited = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdtr['cited_title'],x1)))
x1 = [x+y for x,y in zip(x1,x1_cited)]
y1 = torch.tensor(yc)
x1 = torch.tensor(x1)

print('Oversampling and undersampling======')
#Oversampling the minority classes by SMOTE
tr_over = SMOTE(sampling_strategy={0:1648,1:600,2:600,3:600,4:600,5:600})
#Undersampling the majority class
tr_under = RandomUnderSampler(sampling_strategy={0:600,1:600,2:600,3:600,4:600,5:600})
x1_smote,y1_smote = tr_over.fit_sample(x1,y1)
x1_smote,y1_smote = tr_under.fit_sample(x1_smote,y1_smote)
print(Counter(y1_smote))

x1_smote = torch.tensor(x1_smote)
y1_smote = torch.tensor(y1_smote)
pd.Series(y).value_counts()
# citance + citing title data prep
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
x2 = list(map(lambda t : tokenizer.encode(t),prtr))
x2_citing = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdtr['citing_title'],x2)))
x2 = [x+y for x,y in zip(x2,x2_citing)]
x2 = torch.tensor(x2)
y2 = torch.tensor(yc)
x1 = torch.tensor(x1)

print('Oversampling and undersampling======')
x2_smote,y2_smote = tr_over.fit_sample(x2,y2)
x2_smote,y2_smote = tr_under.fit_sample(x2_smote,y2_smote)
print(Counter(y2_smote))

x2_smote = torch.tensor(x2_smote)
y2_smote = torch.tensor(y2_smote)
(y1_smote == y2_smote).sum()
# citation worthiness data prep
x_cit = datac['cleaned_cite_text'].values.tolist()

ycit = datac['is_citation'].values.tolist()
ycit = list(map(lambda x : int(x),ycit))

# removing instances with token size over 250
xcit_idx_remove_250 = remove_idx(x_cit,250)
for i in sorted(xcit_idx_remove_250, reverse = True):  
    del x_cit[i]
    del ycit[i]
print(f'removed {len(xcit_idx_remove_250)} instances')

xcit = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=250),x_cit))

xcit,ycit = class_div(xcit,ycit,3600,2)

xcit = torch.tensor(xcit)
ycit = torch.tensor(ycit)
# main task data prep
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=250),prtr))
y = torch.tensor(yc)
x = torch.tensor(x)

vx = x[2600:]
vy = y[2600:]
x=x[:2600]
y=y[:2600]
print(x.shape)
print(vx.shape)

print('Oversampling and undersampling======')
tr_over = SMOTE(sampling_strategy={0:1447,1:600,2:600,3:600,4:600,5:600})
tr_under = RandomUnderSampler(sampling_strategy={0:600,1:600,2:600,3:600,4:600,5:600})
x_smote,y_smote = tr_over.fit_sample(x,y)
x_smote,y_smote = tr_under.fit_sample(x_smote,y_smote)
print(Counter(y_smote))

val_over = SMOTE(sampling_strategy={0:201,1:100,2:100,3:100,4:100,5:100},k_neighbors=3)
val_under = RandomUnderSampler(sampling_strategy={0:100,1:100,2:100,3:100,4:100,5:100})
vx_smote,vy_smote = val_over.fit_sample(vx,vy)
vx_smote,vy_smote = val_under.fit_sample(vx_smote,vy_smote)
print(Counter(vy_smote))

x_smote = torch.tensor(x_smote)
y_smote = torch.tensor(y_smote)
vx_smote = torch.tensor(vx_smote)
vy_smote = torch.tensor(vy_smote)
batchSize = 16
num_of_feedforwards = 4
main_tr = torch.utils.data.TensorDataset(x_smote, y_smote)
main_val = torch.utils.data.TensorDataset(vx_smote, vy_smote)
cited_tr = torch.utils.data.TensorDataset(x1_smote,y1_smote)
citing_tr = torch.utils.data.TensorDataset(x2_smote,y2_smote)
citwor_tr = torch.utils.data.TensorDataset(xcit,ycit)
 
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchSize//num_of_feedforwards)

cited_sampler = torch.utils.data.RandomSampler(cited_tr)
cited_data = torch.utils.data.DataLoader(cited_tr, sampler=cited_sampler, batch_size=batchSize//num_of_feedforwards)

citwor_sampler = torch.utils.data.RandomSampler(citwor_tr)
citwor_data = torch.utils.data.DataLoader(citwor_tr, sampler=citwor_sampler, batch_size=batchSize//num_of_feedforwards)

val_sampler = torch.utils.data.RandomSampler(main_val)
val_data = torch.utils.data.DataLoader(main_val, sampler=val_sampler, batch_size=batchSize//num_of_feedforwards)

citing_sampler = torch.utils.data.RandomSampler(citing_tr)
citing_data = torch.utils.data.DataLoader(citing_tr, sampler=citing_sampler, batch_size=batchSize//num_of_feedforwards)
class attention(nn.Module):
    def __init__(self,batch_size,hidden_size=50,max_len=250):
        super().__init__()
        k = torch.empty(batch_size,2*hidden_size,1)
        self.w = torch.nn.init.xavier_uniform_(k).to(device)
        self.hidden = hidden_size
        self.max_len = max_len
    def forward(self,x):
        #e = torch.mm(x.view(-1,2*hidden_size),self.w).view(-1,max_len,1)
        # x.shape = (batch_size,max_len,2*hidden_size), w.shape = (2*hidden_size,1)
        # e.shape = (batch_size,max_len,1)
        # later on also try with w.shape = (batch_size,2*hidden,1) ; use torch.bmm(x,w) in that case instead of above np.dot implementation
        e = torch.bmm(x,self.w)/math.sqrt(self.hidden)
        att_weights = F.softmax(e,dim=1) # attention weights shape = (batch_size,max_len,1)
        out = torch.bmm(x.transpose(1,2),att_weights)
        return out
#feed forward of main task
class feedforward1(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,32)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(32,6)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
#feed forward of citation worthiness scaffold
class feedforward2(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,32)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(32,2)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
#feed forward of citance+cited title scaffold
class feedforward3(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,32)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(32,6)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
#feed forward of citance+citing scaffold
class feedforward4(nn.Module):
    def __init__(self):
        super().__init__()
        self.drop = nn.Dropout(p=0.3)
        self.lin = nn.Linear(100,32)
        self.relu = torch.nn.ReLU()
        self.out = nn.Linear(32,6)
    def forward(self,x):
        x = self.drop(x)
        x = self.lin(x)
        x = self.relu(x)
        x = self.out(x)
        return x 
class MTL_model(nn.Module):
    def __init__(self,batch_size):
        super().__init__()
        self.batch_size = batch_size
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
        self.att = attention(batch_size)
        self.main = feedforward1()
        self.citwor = feedforward2()
        self.cited = feedforward3()
        self.citing = feedforward4()
        
    def forward(self,x,n):
        xbert=self.BertModel(x)
        lstm,_=self.lstm(xbert[0])
        z = self.att(lstm).view(self.batch_size,100)
        if(n==1):
            return self.main(z)
        
        elif(n==2):
            return self.citwor(z)
        
        elif(n==3):
            return self.cited(z)
        elif(n==4):
            return self.citing(z)
        else:
            #predicting == training done!!
            z = self.main(z)
            z = F.softmax(z,dim=1)
            return torch.argmax(z,dim=1)
        
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
def train_model(batchSize,num_of_feedforwards,l1=0.1,l2=0.05,l3=0.05,mod=None):
    torch.manual_seed(-1)
    #training for models
    if(mod==None):    
        mod=MTL_model(batchSize//num_of_feedforwards)
        mod.to(device)

    lambd1 = l1   #lambd1 for citation worthiness scaffold
    lambd2 = l2    #lambd2 for citance+cited title scaffold
    lambd3 = l3    #lambd3 for citance+citing title scaffold

    loss = torch.nn.CrossEntropyLoss()
    optimizer = torch.optim.SGD(mod.parameters(), lr = 0.01)

    loss_list = []
    f1_list = []
    acc_list=[]
    for epoch in range(20):
        running_loss = 0
        ypr=[]
        ytr=[]
        y_eval=[]
        ypr_eval=[]

    # training--------------------
        mod.train()
        for i,data in enumerate(zip(train_data,citwor_data,cited_data,citing_data)):
            m = data[0]
            cw = data[1]
            cted = data[2]
            cting = data[3]

            in_main,tar_main = m[0].to(device),m[1].to(device)
            in_citwor,tar_citwor = cw[0].to(device),cw[1].to(device)
            in_cited,tar_cited = cted[0].to(device),cted[1].to(device)
            in_citing,tar_citing = cting[0].to(device),cting[1].to(device)

            optimizer.zero_grad()

            main = mod(in_main,1)
            citwor = mod(in_citwor,2)
            cited = mod(in_cited,3)
            citing = mod(in_citing,4)

            loss_main = loss(main,tar_main)
            loss_citwor = loss(citwor,tar_citwor)
            loss_cited = loss(cited,tar_cited)
            loss_citing = loss(citing,tar_citing)

            overall_loss = (loss_main + lambd1*loss_citwor + lambd2*loss_cited + lambd3*loss_citing)/num_of_feedforwards  # becoz initially the summation is avg loss per mini batch(8) but we need avg loss per mini batch(24)
            overall_loss.backward()
            torch.nn.utils.clip_grad_norm_(mod.parameters(), 1)
            optimizer.step()

            running_loss += overall_loss.item()
            if(i%100 == 99):
                loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(running_loss / 100,3)})
                print('[epoch : %d,batch : %5d] loss: %.4f' %(epoch + 1, i + 1, running_loss / 100))
                running_loss = 0.0

    # validation ------------------------
        with torch.no_grad():
            mod.eval()

            # calculating f1_score for train data
            for d in train_data:
                x = d[0].to(device)
                y = d[1].to(device)
                for yt in y.cpu():
                    ytr.append(yt)
                y_pred = mod(x,0).cpu()
                for yt in y_pred:
                    ypr.append(yt)

            f1 = f1_score(ytr,ypr,average='macro')
            accuracy = accuracy_score(ytr,ypr)
            # calculating f1_score for validation data
            for d in val_data:
                xv = d[0].to(device)
                yv = d[1].to(device)
                for yt in yv.cpu():
                    y_eval.append(yt)
                y_pred = mod(xv,0).cpu()
                for yt in y_pred:
                    ypr_eval.append(yt)

            val_f1 = f1_score(y_eval,ypr_eval,average='macro')
            val_accuracy = accuracy_score(y_eval,ypr_eval)
            acc_list.append({'epoch':epoch+1,'accuracy':accuracy,'val_accuracy':val_accuracy})
            f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})

        print('*'*40)
        print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f, accuracy: %.3f, val_accuracy: %.3f' %(epoch+1, f1, val_f1, accuracy, val_accuracy))
        print('*'*40)
        print('train confusion matrix : ')
        print(confusion_matrix(ytr, ypr))
        print('*'*40)
        print('val confusion matrix : ')
        print(confusion_matrix(y_eval, ypr_eval))
        print('*'*40)
        print('*'*40)
        if((epoch+1)%5==0):
            torch.save(mod, f'./MTL_model_ep{epoch+1}.pt')

    print('Finished Training!!')    
    return (mod,loss_list,f1_list,acc_list)
from sklearn.model_selection import GridSearchCV
params = {
    'l1': [0.1,0.05,0.2],
    'l2': [0.1,0.05,0.2],
    'l3': [0.1,0.05,0.2]
}
gs = GridSearchCV(train_model(batchSize,num_of_feedforwards), params, refit=False, scoring='f1', verbose=1, cv=5)

model,loss_list,f1_list,acc_list = train_model(batchSize,num_of_feedforwards)
ep = [i['epoch'] for i in f1_list]
train_f1 = [i['train_f1_score'] for i in f1_list]
val_f1 = [i['val_f1_score'] for i in f1_list]
loss = [i['loss'] for i in loss_list]
plt.plot(list(range(180)),loss)
import matplotlib.pyplot as plt
plt.plot(ep,train_f1)
plt.plot(ep,val_f1)
model = torch.load('../input/mlt-model/MTL_model_ep15_ml250.pt')
model.eval()
ytr=[]
ypr=[]
with torch.no_grad():
    model.eval()
# calculating f1_score for train data
    for d in val_data:
        x = d[0].to(device)
        y = d[1].to(device)
        for yt in y.cpu():
            ytr.append(yt)
        y_pred = model(x,0).cpu()
        for yt in y_pred:
            ypr.append(yt)
print('acc : ',accuracy_score(ytr,ypr))
print('f1 : ',f1_score(ytr,ypr,average='macro'))
y
# Submission
prte = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdte['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()[:1000]
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=250),prte))
y = torch.tensor(yc)
x = torch.tensor(x)

print(x.shape)

main_te = torch.utils.data.TensorDataset(x, y)
test_data = torch.utils.data.DataLoader(main_te, batch_size=4)

ind = list(pdte['unique_id'])

pred=[]
ypr=[]
with torch.no_grad():
    model.eval()
    for d in test_data:
        x = d[0].to(device) 
        y_pred = model(x,0).cpu()
        for yt in y_pred:
            ypr.append(yt)
    for j in range(len(ypr)):
        l = [ind[j],ypr[j].item()]
        pred.append(l)
df = pd.DataFrame(pred, columns = ['unique_id', 'citation_class_label']) 
df.set_index('unique_id', inplace = True)

print(df)
df.to_csv('./submission_MTL_model.csv') 
df['citation_class_label'].value_counts()
ind<jupyter_output><empty_output><jupyter_text>**Exp - 2 Representations Model**<jupyter_code>simp_model = torch.load('../input/simple-model/simple_model_ep30_ml250.pt')
simp_model.eval()
cohan_mod = torch.load('../input/latest-model-300/full_model_v1_epoch7_ml300.pt')
cohan_mod.eval()
# code for combining citance and cited title to feed to simp_model

prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t),prtr))
x_cited = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdtr['cited_title'],x)))
x1 = [x+y for x,y in zip(x,x_cited)]
y1 = torch.tensor(yc)
x1 = torch.tensor(x1)
# code to concatenate the (cohan prediction on 3c data) with (citance of 3c data)

yc = pdtr['citation_class_label'].values.tolist()
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),prtr))
y = torch.tensor(yc)
x = torch.tensor(x)

batchsize = 4
main_tr = torch.utils.data.TensorDataset(x,y)
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchsize)

x = list(map(lambda t : tokenizer.encode(t),prtr))
y_cohan = []
with torch.no_grad():
    cohan_mod.eval()
    for d in train_data:
        x_con = d[0].to(device)
        y_con = cohan_mod(x_con,0,'sci')
        for i in y_con.cpu():
            y_cohan.append(str(i.item()))
x2 = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(y_cohan,x)))
x2 = [x+y for x,y in zip(x,x2)]
x2 = torch.tensor(x2)
x1.shape
vx1 = x1[2600:]
x1=x1[:2600]

vx2 = x2[2600:]
x2=x2[:2600]

vy = y[2600:]
y=y[:2600]

print(x1.shape)
print(x2.shape)
print(vx1.shape)
print(vx2.shape)
print(vy.shape)
print(y.shape)
tr_over = SMOTE(sampling_strategy={0:1447,1:600,2:600,3:600,4:600,5:600})
tr_under = RandomUnderSampler(sampling_strategy={0:600,1:600,2:600,3:600,4:600,5:600})
x1_smote,y_smote = tr_over.fit_sample(x1,y)
x1_smote,y_smote = tr_under.fit_sample(x1_smote,y_smote)

x2_smote,y_smote = tr_over.fit_sample(x2,y)
x2_smote,y_smote = tr_under.fit_sample(x2_smote,y_smote)
print(Counter(y_smote))

val1_over = SMOTE(sampling_strategy={0:201,1:100,2:100,3:100,4:100,5:100},k_neighbors=3)
val1_under = RandomUnderSampler(sampling_strategy={0:100,1:100,2:100,3:100,4:100,5:100})
vx1_smote,vy_smote = val1_over.fit_sample(vx1,vy)
vx1_smote,vy_smote = val1_under.fit_sample(vx1_smote,vy_smote)

vx2_smote,vy_smote = val1_over.fit_sample(vx2,vy)
vx2_smote,vy_smote = val1_under.fit_sample(vx2_smote,vy_smote)
print(Counter(vy_smote))

x1_smote = torch.tensor(x1_smote)
x2_smote = torch.tensor(x2_smote)
y_smote = torch.tensor(y_smote)
vx1_smote = torch.tensor(vx1_smote)
vx2_smote = torch.tensor(vx2_smote)
vy_smote = torch.tensor(vy_smote)
batchsize = 4
main_tr1 = torch.utils.data.TensorDataset(x1_smote, y_smote)
main_val1 = torch.utils.data.TensorDataset(vx1_smote, vy_smote)
train_sampler1 = torch.utils.data.RandomSampler(main_tr1)
train_data1 = torch.utils.data.DataLoader(main_tr1, sampler=train_sampler1, batch_size=batchsize)
val_sampler1 = torch.utils.data.RandomSampler(main_val1)
val_data1 = torch.utils.data.DataLoader(main_val1, sampler=val_sampler1, batch_size=batchsize)

main_tr2 = torch.utils.data.TensorDataset(x2_smote, y_smote)
main_val2 = torch.utils.data.TensorDataset(vx2_smote, vy_smote)
train_sampler2 = torch.utils.data.RandomSampler(main_tr2)
train_data2 = torch.utils.data.DataLoader(main_tr2, sampler=train_sampler2, batch_size=batchsize)
val_sampler2 = torch.utils.data.RandomSampler(main_val2)
val_data2 = torch.utils.data.DataLoader(main_val2, sampler=val_sampler2, batch_size=batchsize)
class rep_model(nn.Module):
    def __init__(self,batch_size):
        super().__init__()
        self.batch_size = batch_size
        self.simple_model = simp_model
        for param in self.simple_model.parameters():
            param.requires_grad = False
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
        self.att = attention(batch_size)
        self.relu = torch.nn.ReLU()
        self.drop = nn.Dropout(p=0.3)
        self.lin1 = nn.Linear(200,64)
        self.lin2 = nn.Linear(64,32)
        self.lin3 = nn.Linear(32,6)
        
    def forward(self,x1,x2):
        rep1 = self.simple_model(x1)[1]
        
        xbert=self.BertModel(x2)
        lstm,_=self.lstm(xbert[0])
        rep2 = self.att(lstm).view(self.batch_size,100)
        
        final_rep = torch.cat((rep1,rep2),1).view(self.batch_size,-1)
        
        x = self.drop(final_rep)
        x = self.lin1(x)
        x = self.relu(x)
        x = self.lin2(x)
        x = self.relu(x)
        x = self.lin3(x)
        
        return x
for i,t in enumerate(zip(train_data1,train_data2)):
    data1 = t[0]
    data2 = t[1]
    x1 = data1[0]
    x2 = data2[0]
    y = data1[1]
    print(x1.shape)
    print(i)
    break
x2[0]
model = rep_model(batchsize)
model.to(device)
loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adadelta(model.parameters(), lr = 0.01)
loss_list = []
f1_list = []
acc_list=[]

for epoch in range(25):
    total_loss = 0
    ypr=[]
    ytr=[]
    y_eval=[]
    ypr_eval=[]
    
    model.train()
    
    for i,t in enumerate(zip(train_data1,train_data2)):
        data1 = t[0]
        data2 = t[1]
        
        x1 = data1[0].to(device)
        x2 = data2[0].to(device)
        y = data1[1].to(device)
        
        optimizer.zero_grad()
        pred = model(x1,x2)
        loss_batch = loss(pred,y)
        loss_batch.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25)
        optimizer.step()
        total_loss += (loss_batch).item()
        
        if(i%100 == 99):
            loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(total_loss / 100,3)})
            print('[epoch : %d,batch : %5d] loss: %.3f' %(epoch + 1, i + 1, total_loss / 100))
            total_loss = 0.0
            
    with torch.no_grad():
        model.eval()
        simp_model.eval()
        
        # calculating f1_score for train data
        for d in zip(train_data1,train_data2):
            x1 = d[0][0].to(device)
            x2 = d[1][0].to(device)
            y = d[0][1].to(device)
            
            for yt in y.cpu():
                ytr.append(yt)
                
            z = model(x1,x2)
            z = F.softmax(z,dim=1)
            z = torch.argmax(z,dim=1)    
            y_pred = z.cpu()
            for yt in y_pred:
                ypr.append(yt)
                
        f1 = f1_score(ytr,ypr,average='macro')
        accuracy = accuracy_score(ytr,ypr)
        
        # calculating f1_score for validation data
        for d in zip(val_data1,val_data2):
            xv1 = d[0][0].to(device)
            xv2 = d[1][0].to(device)
            yv = d[0][1].to(device)
            
            for yt in yv.cpu():
                y_eval.append(yt)
            
            z = model(xv1,xv2)
            z = F.softmax(z,dim=1)
            z = torch.argmax(z,dim=1)    
            y_pred = z.cpu()
            for yt in y_pred:
                ypr_eval.append(yt)
                
        val_f1 = f1_score(y_eval,ypr_eval,average='macro')  
        val_accuracy = accuracy_score(y_eval,ypr_eval)
        acc_list.append({'epoch':epoch+1,'accuracy':accuracy,'val_accuracy':val_accuracy})
        f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})
    
    target_names = ['class 0', 'class 1', 'class 2', 'class 3', 'class 4', 'class 5']
    print('*'*40)
    print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f, accuracy: %.3f, val_accuracy: %.3f' %(epoch+1, f1, val_f1, accuracy, val_accuracy))
    print('*'*40)
    print('train confusion matrix : ')
    print(confusion_matrix(ytr, ypr))
    print('*'*40)
    print('val confusion matrix : ')
    print(confusion_matrix(y_eval, ypr_eval))
    print('*'*40)
    print('train classification report : ')
    print(classification_report(ytr, ypr, target_names=target_names))
    print('*'*40)
    print('test classification report : ')
    print(classification_report(y_eval,ypr_eval, target_names=target_names))
    print('*'*40)
    if((epoch+1)%5==0):
        torch.save(model, f'./simple_model_ep{epoch+1}.pt')

print('Finished Training!!') 
x.size(0)
y.size(0)
model = torch.load('./simple_model_ep15.pt')
model.eval()
### Submission file generation

batchsize = 4
prte = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdte['citation_context']))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx1 = list(map(lambda t : tokenizer.encode(t),prte))
tx1_cited = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(pdte['cited_title'],tx1)))
tx1 = [x+y for x,y in zip(tx1,tx1_cited)]
tx1 = torch.tensor(tx1)
print(tx1[0])

# code to concatenate the (cohan prediction on 3c data) with (citance of 3c data)

yc = pdtr['citation_class_label'].values.tolist()[:1000]   # any random value of y taken so that we can use the TensorDataset for ease in coding
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),prte))
y = torch.tensor(yc)
x = torch.tensor(x)

batchsize = 4
main_te = torch.utils.data.TensorDataset(x,y)
test_sampler = torch.utils.data.RandomSampler(main_te)
test_data = torch.utils.data.DataLoader(main_te, sampler=test_sampler, batch_size=batchsize)

x = list(map(lambda t : tokenizer.encode(t),prte))
y_cohan = []
with torch.no_grad():
    cohan_mod.eval()
    for d in test_data:
        x_con = d[0].to(device)
        y_con = cohan_mod(x_con,0,'sci')
        for i in y_con.cpu():
            y_cohan.append(str(i.item()))
x2 = list(map(lambda t : tokenizer.encode(t[0],padding=True,pad_to_max_length=True,max_length=251-len(t[1]))[1:],zip(y_cohan,x)))
tx2 = [x+y for x,y in zip(x,x2)]
tx2 = torch.tensor(tx2)
print('tx2 shape : ',tx2.shape)

ind = list(pdte['unique_id'])

pred = []
with torch.no_grad():
    model.eval()
    for i in range(0,len(tx1),4):
        l=[]
        x1 = tx1[i:i+4]
        x2 = tx2[i:i+4]
        idx = ind[i:i+4]
        x1 = x1.to(device)
        x2 = x2.to(device)
        y_pr = model(x1,x2).cpu()
        y_pr = F.softmax(y_pr,dim=1)
        y_pr = torch.argmax(y_pr,dim=1)
        for j in range(len(x1)):
            l = [idx[j],y_pr[j].item()]
            pred.append(l)
        
df = pd.DataFrame(pred, columns = ['unique_id', 'citation_class_label']) 
df.set_index('unique_id', inplace = True)

print(df)
df.to_csv('./submission.csv') 
df['citation_class_label'].value_counts()<jupyter_output><empty_output><jupyter_text>**Exp - 3 Late fusion**<jupyter_code>mod = torch.load('../input/cohanmodel-on-sci-and-acl/full_model_ep25.pt')
mod.eval()
model = torch.load('./model_pk_epoch20.pt')
model.eval()
prtr = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+'( @@CITATION )'+t[t.find('#AUTHOR_TAG')+11:],pdtr['citation_context']))
yc = pdtr['citation_class_label'].values.tolist()
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),prtr))
y = torch.tensor(yc)
x = torch.tensor(x)
print(x.shape)
print(y.shape)
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
acl_tr = list(acl_pdm['cleaned_cite_text'])
acl_y = acl_pdm['intent'].values.tolist()
acl_label={'Background':0,'CompareOrContrast':1,'Extends':2,'Future':3,'Motivation':4,'Uses':5}
acl_y = list(map(lambda t : acl_label[t],acl_y))

print(acl_y[:5])

acl_tr_idx_remove_300 = remove_idx(acl_tr,300)
for i in sorted(acl_tr_idx_remove_300, reverse = True):  
    del acl_tr[i]
    del acl_y[i]
print(f'removed {len(acl_tr_idx_remove_300)} instances')
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
acl_x = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),acl_tr))
acl_y = torch.tensor(acl_y)
acl_x = torch.tensor(acl_x)
print(acl_x[:5])
print(acl_x.shape)
print(acl_y[:5])
print(acl_y.shape)
x_tr = torch.cat((x,acl_x),0)
y_tr = torch.cat((y,acl_y),0)
print(x_tr.shape)
print(y_tr.shape)
from sklearn.model_selection import train_test_split
x,vx,y,vy = train_test_split(x, y, test_size=400/3000, random_state=42,shuffle=True)
print(x.shape)
print(vx.shape)
print(y.shape)
print(vy.shape)
pd.Series(y).value_counts()
pd.Series(vy).value_counts()
yall = [i.item() for i in y]
Counter(yall)
yallv = [i.item() for i in vy]
Counter(yallv)
# only oversampling of minority classes done for now!!!
sampling_strategy = {0:2126,1:600,2:600,3:600,4:600,5:600}  # use this if u wnat to undersample using RandomUnderSampler after oversampling by SMOTE
tr_over = SMOTE(sampling_strategy='not majority')
#tr_under = RandomUnderSampler(sampling_strategy={0:600,1:600,2:600,3:600,4:600,5:600})
x_smote,y_smote = tr_over.fit_sample(x,y)
#x_smote,y_smote = tr_under.fit_sample(x_smote,y_smote)
Counter(y_smote)
sampling_strategy = {0:201,1:100,2:100,3:100,4:100,5:100} # use this if u want to undersample as well after oversampling by SMOTE is done
# while using above dict for sampling strategy of SMOTE, put k_neighbors = 3
val_over = SMOTE(sampling_strategy='not majority')
#val_under = RandomUnderSampler(sampling_strategy={0:100,1:100,2:100,3:100,4:100,5:100})
vx_smote,vy_smote = val_over.fit_sample(vx,vy)
#vx_smote,vy_smote = val_under.fit_sample(vx_smote,vy_smote)
Counter(vy_smote)
x_smote = torch.tensor(x_smote)
y_smote = torch.tensor(y_smote)
vx_smote = torch.tensor(vx_smote)
vy_smote = torch.tensor(vy_smote)
y_smote.shape
vy_smote.shape
# to maintain every batch with batch_size = 4 => we leave the last 2 instances...........
# vy_smote = vy_smote[:-2]
# vx_smote = vx_smote[:-2]
# print(vy_smote.shape)

batchsize = 4
main_tr = torch.utils.data.TensorDataset(x_smote, y_smote)
main_val = torch.utils.data.TensorDataset(vx_smote, vy_smote)
train_sampler = torch.utils.data.RandomSampler(main_tr)
train_data = torch.utils.data.DataLoader(main_tr, sampler=train_sampler, batch_size=batchsize)
val_sampler = torch.utils.data.RandomSampler(main_val)
val_data = torch.utils.data.DataLoader(main_val, sampler=val_sampler, batch_size=batchsize)
os.mkdir('./glove')
from gensim.scripts.glove2word2vec import glove2word2vec
glove_input_file = '../input/glove-pretrained/glove.6B.100d.txt'
word2vec_output_file = './glove/glove.6B.100d.txt.word2vec'
glove2word2vec(glove_input_file, word2vec_output_file)
from gensim.models import KeyedVectors
# load the Stanford GloVe model
filename = './glove/glove.6B.100d.txt.word2vec'
glove = KeyedVectors.load_word2vec_format(filename, binary=False)
print(glove)
# # calculate: (king - man) + woman = ?
# result = model.most_similar(positive=['woman', 'king'], negative=['man'], topn=1)
# print(result)
required_vocab = ['introduction','conclusion','experiments','method','related','work','true','false','background','result']
embedding_matrix
first=0
embedding_matrix = dict()
for i in required_vocab:
    if i=='related' or i=='work':
        if(first):
            second = i
            if(first=='related'):
                string = first + ' ' + second
            else:
                string = second + ' ' + first
            embedding_matrix[string] = (glove[first]+glove[second])/2
        else:    
            first = i
    else:
        embedding_matrix[i]=glove[i]
'introduction':0,'related work':1,'method':2,'experiments':3,'conclusion':4
label_dict = {0:'background',1:'method',2:'result'}
sec_label_dict = {0:'introduction',1:'related work',2:'method',3:'experiments',4:'conclusion'} 
cit_label_dict = {0:'false',1:'true'}
sec_label_dict[3.]
# external knowledge (predictions of label,citation worthiness,section of citation) from pre trained cohan model for simple fusion model

ycit=[]
ysec=[]
ypredict=[]
ycitv=[]
ysecv=[]
ypredictv=[]
with torch.no_grad():
    mod.eval()
    for t in train_data:                       # pride knoth data
        xt = t[0].to(device)
        y_pred = mod(xt,0,'sci').cpu()
        y_sec = torch.argmax(F.softmax(mod(xt,2),dim=1),dim=1).cpu()
        y_cit_wor = torch.argmax(F.softmax(mod(xt,3),dim=1),dim=1).cpu()
        for ypr,ys,yc in zip(y_pred,y_sec,y_cit_wor):
            label = label_dict[ypr.item()]
            sec_label=sec_label_dict[ys.item()]
            cit_label=cit_label_dict[yc.item()]
            ypredict.append(label)
            ysec.append(sec_label)
            ycit.append(cit_label)
# ypredict = torch.tensor(ypredict).view(2600,1,-1).float().cuda()
# ysec = torch.tensor(ysec).view(2600,1,-1).float().cuda()
# ycit = torch.tensor(ycit).view(2600,1,-1).float().cuda()
# external knowledge (last dense layer representation) from pre trained cohan model for simple fusion model

ypredict=[]
ypredictv=[]
with torch.no_grad():
    mod.eval()
    for t in train_data:                       # pride knoth data
        xt = t[0].to(device)
        y_pred = mod(xt,0,'pk')[4].cpu()
        for ypr in y_pred:
            ypredict.append(ypr.view(1,20,-1))
    for t in val_data:                       # pride knoth data
        xt = t[0].to(device)
        y_pred = mod(xt,0,'pk')[4].cpu()
        for ypr in y_pred:
            ypredictv.append(ypr.view(1,20,-1))
ypredict = torch.cat(ypredict,0).cuda()
ypredictv = torch.cat(ypredictv,0).cuda()
ypredict.view(-1).cpu() 
yall=pd.Series(list(ypredict.view(-1).cpu()))
yall.value_counts()
ysec=pd.Series(ysec)
ysec.value_counts()
ycit=pd.Series(ycit)
ycit.value_counts()
# external knowledge from pre trained cohan model for conv layered fusion model
ycit=[]
ysec=[]
ypredict=[]
ycitv=[]
ysecv=[]
ypredictv=[]

train_num = y_smote.shape[0]
val_num = vy_smote.shape[0]
with torch.no_grad():
    mod.eval()
    for t in train_data:                       # pride knoth data
        xt = t[0].to(device)
        y_pred = mod(xt,0,'sci').cpu()
        y_sec = torch.argmax(F.softmax(mod(xt,2),dim=1),dim=1).cpu()
        y_cit_wor = torch.argmax(F.softmax(mod(xt,3),dim=1),dim=1).cpu()
        for ypr,ys,yc in zip(y_pred,y_sec,y_cit_wor):
            label = label_dict[ypr.item()]
            sec_label=sec_label_dict[ys.item()]
            cit_label=cit_label_dict[yc.item()]
            
            ypr_embed = embedding_matrix[label]
            ys_embed = embedding_matrix[sec_label]
            yc_embed = embedding_matrix[cit_label]
            ypredict.append(ypr_embed)
            ysec.append(ys_embed)
            ycit.append(yc_embed)
            
    for t in val_data:    
        xv = t[0].to(device)    
        y_predv = mod(xv,0,'sci').cpu()
        y_secv = torch.argmax(F.softmax(mod(xv,2),dim=1),dim=1).cpu()
        y_cit_worv = torch.argmax(F.softmax(mod(xv,3),dim=1),dim=1).cpu()
        for ypr,ys,yc in zip(y_predv,y_secv,y_cit_worv):
            label = label_dict[ypr.item()]
            sec_label=sec_label_dict[ys.item()]
            cit_label=cit_label_dict[yc.item()]
            
            ypr_embed = embedding_matrix[label]
            ys_embed = embedding_matrix[sec_label]
            yc_embed = embedding_matrix[cit_label]
            ypredictv.append(ypr_embed)
            ysecv.append(ys_embed)
            ycitv.append(yc_embed)
            
ypredict = torch.tensor(ypredict).view(train_num,100,-1).float().cuda()
ysec = torch.tensor(ysec).view(train_num,100,-1).float().cuda()
ycit = torch.tensor(ycit).view(train_num,100,-1).float().cuda()

ypredictv = torch.tensor(ypredictv).view(val_num,100,-1).float().cuda()
ysecv = torch.tensor(ysecv).view(val_num,100,-1).float().cuda()
ycitv = torch.tensor(ycitv).view(val_num,100,-1).float().cuda()
ypredictv.shape
ysec[0]
y1=ypredict[:4]
y2=ysec[:4]
y3=ycit[:4]
cat = torch.cat((y1,y2,y3),2)
cat.shape
yel.shape
yel = torch.rand(4,100,1).cuda()
cate = torch.cat((yel,cat),2)
cate.shape
for i in ysec:
    print(i)
yprel = torch.rand(4,1,1)
ysect = torch.rand(4,1,1)
yw = torch.rand(4,1,1)
yel = torch.rand(4,100,1)
print(ysect)
print(yw)
print(yprel)
print(yel)
cat = torch.cat((yel,yprel,ysect,yw),1)
class Modelfusion(nn.Module):
    def __init__(self,batch_size,fus):
        super().__init__()
        self.batch_size = batch_size
        self.fus = fus                    # no. of different params(like pred. label,section,cit worthiness by pre trained cohan model) we are fusing with the output after attention layer.
        self.n = 100 + fus
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
        self.att = attention(batch_size)
        self.lin1 = nn.Linear(self.n,64)
        self.relu = torch.nn.ReLU()
        self.drop = nn.Dropout(p=0.4)
        self.lin2 = nn.Linear(64,6)
        
    def forward(self,x,xfus):
        xbert=self.BertModel(x)
        lstm,_=self.lstm(xbert[0])
        z = self.att(lstm)
        z = torch.cat((z,xfus),1).view(self.batch_size,self.n)
        z = self.drop(z)
        z = self.lin1(z)
        z = self.relu(z)
        z = self.lin2(z)
        return z
# just for checking iutput dimensions
for i in train_data:
    xall = i[0]
    break
# just for checking iutput dimensions
xall[0].shape
# just for checking iutput dimensions
bert = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")(xall[0].view(1,-1))
lst,_ = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)(bert[0])
at = attention(1)(lst.to(device))
at.shape
# just for checking iutput dimensions
xf = torch.rand(1,20,1).cuda()
torch.cat((at,xf),1).view(1,120).shape
class Modelfusion_embed(nn.Module):
    def __init__(self,batch_size,fus):
        super().__init__()
        self.batch_size = batch_size
        self.fus = fus                    # no. of different params(like pred. label,section,cit worthiness by pre trained cohan model) we are fusing with the output after attention layer.
        self.n = 100 + fus
        self.BertModel = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
        self.lstm = nn.LSTM(768,50,num_layers=1,bidirectional=True,batch_first=True)
        self.att = attention(batch_size)
        self.cnn1 = nn.Conv1d(4,2,2,stride=2)
        self.max = nn.MaxPool1d(2, stride=2)
        self.lin1 = nn.Linear(50,16)
        self.relu = torch.nn.ReLU()
        self.drop = nn.Dropout(p=0.4)
        self.lin2 = nn.Linear(16,6)
        
    def forward(self,x,xfus):
        xbert=self.BertModel(x)
        lstm,_=self.lstm(xbert[0])
        z = self.att(lstm)
        z = torch.cat((z,xfus),2).permute(0,2,1)
        z = self.cnn1(z)
        z = self.relu(z)
        z = self.max(z)
        z = self.lin1(z.view(self.batch_size,-1))
        z = self.relu(z)
        z = self.drop(z)
        z = self.lin2(z)
        return z
m = nn.Conv1d(4, 2, 2, stride=2)
mx = nn.MaxPool1d(2, stride=2)
input = torch.randn(4, 4, 100)
output = m(input)
print(output.shape)
output = mx(output)
print(output.shape)
os.mkdir('./models')
fus.shape
# evaluation of model before training...........

model = Modelfusion_embed(batchsize,3)
model.to(device)
ypr=[]
ytr=[]
y_eval=[]
ypr_eval=[]
with torch.no_grad():
    model.eval()
        
    # calculating f1_score for train data
    for i,t in enumerate(train_data):
        x = t[0].to(device)
        y = t[1].to(device)
        for yt in y.cpu():
            ytr.append(yt)
                
        y1 = ypredict[i*batchsize:(i+1)*batchsize]
        y2 = ysec[i*batchsize:(i+1)*batchsize]
        y3 = ycit[i*batchsize:(i+1)*batchsize]
        fus = torch.cat((y1,y2,y3),2)    
        z = model(x,fus)
        z = F.softmax(z,dim=1)
        z = torch.argmax(z,dim=1)    
        y_pred = z.cpu()
        for yt in y_pred:
            ypr.append(yt)
                
    f1 = f1_score(ytr,ypr,average='macro')
    for i,t in enumerate(val_data):
        xv = t[0].to(device)
        yv = t[1].to(device)
        for yt in yv.cpu():
            y_eval.append(yt)
            
        y1 = ypredictv[i*batchsize:(i+1)*batchsize]
        y2 = ysecv[i*batchsize:(i+1)*batchsize]
        y3 = ycitv[i*batchsize:(i+1)*batchsize]
        fus = torch.cat((y1,y2,y3),2) 
        z = model(xv,fus)
        z = F.softmax(z,dim=1)
        z = torch.argmax(z,dim=1)    
        y_pred = z.cpu()
        for yt in y_pred:
            ypr_eval.append(yt)
                
    val_f1 = f1_score(y_eval,ypr_eval,average='macro')  
print('train_f1 score before training : ',f1)
print('val_f1 score before training : ',val_f1)
ytr1 = [i.item() for i in ytr]
pd.Series(ytr1).value_counts()
y_eval1 = [i.item() for i in y_eval]
pd.Series(y_eval1).value_counts()
len(ytr1)
ypr1 = [i.item() for i in ypr]
pd.Series(ypr1).value_counts()
ypredict.shape
model = torch.load('./acl_fusionmodel_ep15.pt')
model.eval()
for param in model.parameters():
  print(param.name,param.data)
for name, param in model.named_parameters():
    if(name.split('.')[0]!='BertModel'):
        print(name,param.data)
print('gradients of fusion model before training : ')
for name,params in model.named_parameters():
    print(name)
    print(params.grad)
# training of the model

# model = Modelfusion(batchsize,20)
# model.to(device)
# loss = torch.nn.CrossEntropyLoss()
# optimizer = torch.optim.Adadelta(model.parameters(), lr = 0.01)

# loss_list = []
# f1_list = []
# acc_list=[]

# print('weights before the start of training : ')
# for name, param in model.named_parameters():
#     if(name.split('.')[0]!='BertModel'):
#         print(name,param.data)
        
print('gradients of fusion model last layer before training : ')
print(list(model.named_parameters())[-1][0])
print(list(model.named_parameters())[-1][1].grad)
print(list(model.named_parameters())[-2][0])
print(list(model.named_parameters())[-2][1].grad)
print('*'*80)

for epoch in range(5):
    total_loss = 0
    ypr=[]
    ytr=[]
    y_eval=[]
    ypr_eval=[]
    
    model.train()
    
    print(f'gradients of fusion model before epoch {epoch+1} : ')
    for name,params in model.named_parameters():
        print(name)
        print(params.grad)
        
    print('@'*80)
    
    
    for i,t in enumerate(train_data):
        inp,out = t[0].to(device),t[1].to(device)
        
        optimizer.zero_grad()
        
        y1 = ypredict[i*batchsize:(i+1)*batchsize]
#         y2 = ysec[i*batchsize:(i+1)*batchsize]
#         y3 = ycit[i*batchsize:(i+1)*batchsize]
       # fus = torch.cat((y1,y2,y3),2) 
        fus = y1
        
        pred = model(inp,fus)
        
        loss_batch = loss(pred,out)
        
        loss_batch.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
        optimizer.step()
        total_loss += loss_batch.item()
        
        if(i%100 == 99):
            loss_list.append({'epoch':epoch+1,'batch':i+1,'loss':round(total_loss / 100,3)})
            print('[epoch : %d,batch : %5d] loss: %.3f' %(epoch + 1, i + 1, total_loss / 100))
            total_loss = 0.0
            print('\n')
            print('grads of last layer of fusion model: ')
            print(list(model.named_parameters())[-1][1].grad)
            print('@'*80)
            
    with torch.no_grad():
        model.eval()
        
        # calculating f1_score for train data
        for i,t in enumerate(train_data):
            x = t[0].to(device)
            y = t[1].to(device)
            for yt in y.cpu():
                ytr.append(yt)
                
            y1 = ypredict[i*batchsize:(i+1)*batchsize]
#             y2 = ysec[i*batchsize:(i+1)*batchsize]
#             y3 = ycit[i*batchsize:(i+1)*batchsize]
#             fus = torch.cat((y1,y2,y3),2) 
            fus = y1
            z = model(x,fus)
            z = F.softmax(z,dim=1)
            z = torch.argmax(z,dim=1)    
            y_pred = z.cpu()
            for yt in y_pred:
                ypr.append(yt)
                
        f1 = f1_score(ytr,ypr,average='macro')
        accuracy = accuracy_score(ytr,ypr)
        
        # calculating f1_score for validation data
        for i,t in enumerate(val_data):
            xv = t[0].to(device)
            yv = t[1].to(device)
            for yt in yv.cpu():
                y_eval.append(yt)
            
            y1 = ypredictv[i*batchsize:(i+1)*batchsize]
#             y2 = ysecv[i*batchsize:(i+1)*batchsize]
#             y3 = ycitv[i*batchsize:(i+1)*batchsize]
#             fus = torch.cat((y1,y2,y3),2) 
            fus =y1
            z = model(xv,fus)
            z = F.softmax(z,dim=1)
            z = torch.argmax(z,dim=1)    
            y_pred = z.cpu()
            for yt in y_pred:
                ypr_eval.append(yt)
                
        val_f1 = f1_score(y_eval,ypr_eval,average='macro')  
        val_accuracy = accuracy_score(y_eval,ypr_eval)
        acc_list.append({'epoch':epoch+1,'accuracy':accuracy,'val_accuracy':val_accuracy})
        f1_list.append({'epoch':epoch+1,'train_f1_score':f1,'val_f1_score':val_f1})
    
    print('*'*40)
    print('[epoch : %d] train_f1_macro: %.3f, val_f1_macro: %.3f,  accuracy: %.3f, val_accuracy: %.3f' %(epoch+1, f1, val_f1, accuracy, val_accuracy))
    print('train confusion matrix : ')
    print(confusion_matrix(ytr, ypr))
    print('*'*40)
    print('val confusion matrix : ')
    print(confusion_matrix(y_eval, ypr_eval))
    print('#'*80)
    
    if((epoch+1)%2==0):
        print('weights of the non-Bert layers for epoch {epoch+1} : ')
        for name, param in model.named_parameters():
            if(name.split('.')[0]!='BertModel'):
                print(name,param.data)
        print('#'*80)
            
    if((epoch+1)%5==0):
        torch.save(model, f'./acl_fusionmodel_ep{epoch+1}.pt')

print('Finished Training!!') 
print('weights of fusion model before the start of training : ')
for name, param in model.named_parameters():
    if(name.split('.')[0]!='BertModel'):
        print(name,param.data)
print(f'gradients of cohan model aftre training : ')
for name,params in mod.named_parameters():
    print(name)
    print(params.grad)
print(f'gradients of fusion model aftre training : ')
for name,params in model.named_parameters():
    print(name)
    print(params.grad)
print(list(model.named_parameters())[-1][1].grad)
list(model.parameters())[-1].grad
ep = [i['epoch'] for i in f1_list]
train_f1 = [i['train_f1_score'] for i in f1_list]
val_f1 = [i['val_f1_score'] for i in f1_list]

loss_graph = [i['loss'] for i in loss_list]

import matplotlib.pyplot as plt
plt.plot(ep,train_f1)
plt.plot(ep,val_f1)
plt.plot(list(range(315)),loss_graph)
torch.save(model, f'./acl_fusionmodel_ep15_diff.pt')
y1.shape
import os
os.chdir(r'../working')
from IPython.display import FileLink
FileLink(r'./acl_fusionmodel_ep20.pt')
for param in mod.parameters():
  print(param)
#before training
for param in model.parameters():
  print(param)
#after training
for param in model.parameters():
  print(param)
ypr1 = [i.item() for i in ypr]
pd.Series(ypr1).value_counts()
torch.save(model,'./model_pk_epoch24.pt')
### Submission file generation for last layer representation variant using (ACL + 3C) data 

prte = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+'( @@CITATION )'+t[t.find('#AUTHOR_TAG')+11:],pdte['citation_context']))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),prte))
tx = torch.tensor(tx)

ind = list(pdte['unique_id'])

pred = []
with torch.no_grad():
    model.eval()
    mod.eval()
    for i in range(0,len(tx),4):
        l=[]
        ypredict=[]
        x = tx[i:i+4]
        x = x.to(device)
        if(i==0):
            print(x.shape)
        y_pred = mod(x,0,'sci')[1].cuda()
        
        for ypr in y_pred:
            ypredict.append(ypr.view(1,20,-1))
        
        ypredict = torch.cat(ypredict,0)
        idx = ind[i:i+4]
        fus = ypredict
        if(i==0):
            print(ypredict.shape)
        
        y_pr = model(x,fus)
        y_pr = F.softmax(y_pr,dim=1)
        if(i==0):
            print(y_pr)
        y_pr = torch.argmax(y_pr,dim=1).cpu()
        
        if(i==0):
            print(y_pr)
            
        for j in range(len(x)):
            l = [idx[j],y_pr[j].item()]
            pred.append(l)
            
df = pd.DataFrame(pred, columns = ['unique_id', 'citation_class_label']) 
df.set_index('unique_id', inplace = True)

print(df)
df.to_csv('./submission4.csv') 
### Submission file generation for conv layer variant

prte = list(map(lambda t : t[:t.find('#AUTHOR_TAG')]+t[t.find('#AUTHOR_TAG')+11:],pdte['citation_context']))
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased",do_lower_case=True)
tx = list(map(lambda t : tokenizer.encode(t,padding=True,pad_to_max_length=True,max_length=300),prte))
tx = torch.tensor(tx)

ind = list(pdte['unique_id'])

pred = []
with torch.no_grad():
    model.eval()
    mod.eval()
    for i in range(0,len(tx),4):
        l=[]
        ypredict=[]
        ysec=[]
        ycit=[]
        x = tx[i:i+4]
        x = x.to(device)
        if(i==0):
            print(x.shape)
        y_pred = mod(x,0,'sci').view(4,1,-1).float().cuda()
        y_sec = torch.argmax(F.softmax(mod(x,2),dim=1),dim=1).view(4,1,-1).float().cuda()
        y_cit_wor = torch.argmax(F.softmax(mod(x,3),dim=1),dim=1).view(4,1,-1).float().cuda()
        
        for ypr,ys,yc in zip(y_pred,y_sec,y_cit_wor):
            label = label_dict[ypr.item()]
            sec_label=sec_label_dict[ys.item()]
            cit_label=cit_label_dict[yc.item()]
            
            ypr_embed = embedding_matrix[label]
            ys_embed = embedding_matrix[sec_label]
            yc_embed = embedding_matrix[cit_label]
            ypredict.append(ypr_embed)
            ysec.append(ys_embed)
            ycit.append(yc_embed)
        ypredict = torch.tensor(ypredict).view(4,100,-1).float().cuda()
        ysec = torch.tensor(ysec).view(4,100,-1).float().cuda()
        ycit = torch.tensor(ycit).view(4,100,-1).float().cuda()
        idx = ind[i:i+4]
        fus = torch.cat((ypredict,ysec,ycit),2)
        if(i==0):
            print(ypredict.shape)
            print(ysec.shape)
            print(ycit.shape)
            print(fus.shape)
        
        y_pr = model(x,fus)
        y_pr = F.softmax(y_pr,dim=1)
        print(y_pr)
        y_pr = torch.argmax(y_pr,dim=1).cpu()
        
        if(i==0):
            print(y_pr)
            
        for j in range(len(x)):
            l = [idx[j],y_pr[j].item()]
            pred.append(l)
            
df = pd.DataFrame(pred, columns = ['unique_id', 'citation_class_label']) 
df.set_index('unique_id', inplace = True)

print(df)
df.to_csv('./submission4.csv') 
df['citation_class_label'].value_counts()
df['citation_class_label'].value_counts()
tx[0]
model = Modelfusion(1).cuda()
m = model(tx[0].cuda().view(1,-1))
m.shape
ysec[0].float()
ysec[0].float().cuda().view(1,1,-1)
att.shape
torch.cat((att.view(5,20,-1),ysec[0].float().cuda().view(1,1,-1)), 1)
(att[0][0][0])
att.view(5,20,-1).shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15721586562718706, 'fraction_numerical': 0.029678878924496672, 'mean_word_length': 3.3625035501278044, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '253413', 'license': 'MIT', 'n_tokens_mistral': 49174, 'n_tokens_neox': 47543, 'n_words': 7528, 'path': 'Models/Experiments.ipynb', 'repo_name': 'vkk1710/Citation-Context-Classification', 'size': '122882'}