<jupyter_start><jupyter_text>**Focus Net**<jupyter_code>class Focus_deep(nn.Module):
    '''
       deep focus network averaged at zeroth layer
       input : elemental data
    '''
    def __init__(self,inputs,output,K,d):
        super(Focus_deep,self).__init__()
        self.inputs = inputs
        self.output = output
        self.K = K
        self.d  = d
        self.linear1 = nn.Linear(self.inputs,100)  #,self.output)
        self.linear2 = nn.Linear(100,self.output) 
    def forward(self,z):
        batch = z.shape[0]
        x = torch.zeros([batch,self.K],dtype=torch.float64)
        y = torch.zeros([batch,self.d], dtype=torch.float64)
        x,y = x.to("cuda"),y.to("cuda")
        for i in range(self.K):
            x[:,i] = self.helper(z[:,i] )[:,0]  # self.d*i:self.d*i+self.d
        x = F.softmax(x,dim=1)   # alphas
        x1 = x[:,0]
        for i in range(self.K):
            x1 = x[:,i]          
            y = y+torch.mul(x1[:,None],z[:,i])  # self.d*i:self.d*i+self.d
        return y , x 
    def helper(self,x):
      x = F.relu(self.linear1(x))
      x = self.linear2(x)
      return x<jupyter_output><empty_output><jupyter_text>**Classification Net**<jupyter_code>class Classification_deep(nn.Module):
    '''
       input : elemental data
       deep classification module data averaged at zeroth layer
    '''
    def __init__(self,inputs,output):
        super(Classification_deep,self).__init__()
        self.inputs = inputs
        self.output = output
        self.linear1 = nn.Linear(self.inputs,300)
        self.linear2 = nn.Linear(300,self.output)

    def forward(self,x):
      x = F.relu(self.linear1(x))
      x = self.linear2(x)
      return x<jupyter_output><empty_output><jupyter_text><jupyter_code>where = Focus_deep(5,1,9,5).double()
what = Classification_deep(5,3).double()
where = where.to("cuda")
what = what.to("cuda")
def calculate_attn_loss(dataloader,what,where,criter):
  what.eval()
  where.eval()
  r_loss = 0
  alphas = []
  lbls = []
  pred = []
  fidices = []
  with torch.no_grad():
    for i, data in enumerate(dataloader, 0):
      inputs, labels,fidx = data
      lbls.append(labels)
      fidices.append(fidx)
      inputs = inputs.double()
      inputs, labels = inputs.to("cuda"),labels.to("cuda")
      avg,alpha = where(inputs)
      outputs = what(avg)
      _, predicted = torch.max(outputs.data, 1)
      pred.append(predicted.cpu().numpy())
      alphas.append(alpha.cpu().numpy())

      mx,_ = torch.max(alpha,1)
      entropy = np.mean(-np.log2(mx.cpu().detach().numpy()))
      # print("entropy of batch", entropy)

      loss = criter(outputs, labels) + entropy
      r_loss += loss.item()
  alphas = np.concatenate(alphas,axis=0)
  pred = np.concatenate(pred,axis=0)
  lbls = np.concatenate(lbls,axis=0)
  fidices = np.concatenate(fidices,axis=0)
  #print(alphas.shape,pred.shape,lbls.shape,fidices.shape) 
  analysis = analyse_data(alphas,lbls,pred,fidices)
  return r_loss/i,analysis
def analyse_data(alphas,lbls,predicted,f_idx):
    '''
       analysis data is created here
    '''
    batch = len(predicted)
    amth,alth,ftpt,ffpt,ftpf,ffpf = 0,0,0,0,0,0
    for j in range (batch):
      focus = np.argmax(alphas[j])
      if(alphas[j][focus] >= 0.5):
        amth +=1
      else:
        alth +=1
      if(focus == f_idx[j] and predicted[j] == lbls[j]):
        ftpt += 1
      elif(focus != f_idx[j] and predicted[j] == lbls[j]):
        ffpt +=1
      elif(focus == f_idx[j] and predicted[j] != lbls[j]):
        ftpf +=1
      elif(focus != f_idx[j] and predicted[j] != lbls[j]):
        ffpf +=1
    #print(sum(predicted==lbls),ftpt+ffpt)
    return [ftpt,ffpt,ftpf,ffpf,amth,alth]
print("--"*40)
criterion = nn.CrossEntropyLoss()
optimizer_where = optim.Adam(where.parameters(),lr =0.001)
optimizer_what = optim.Adam(what.parameters(), lr=0.001)
acti = []
loss_curi = []
analysis_data = []
epochs = 1000
running_loss,anlys_data = calculate_attn_loss(train_loader,what,where,criterion)
loss_curi.append(running_loss)
analysis_data.append(anlys_data)
print('epoch: [%d ] loss: %.3f' %(0,running_loss)) 
for epoch in range(epochs): # loop over the dataset multiple times
  ep_lossi = []
  running_loss = 0.0
  what.train()
  where.train()
  for i, data in enumerate(train_loader, 0):
    # get the inputs
    inputs, labels,_ = data
    inputs = inputs.double()
    inputs, labels = inputs.to("cuda"),labels.to("cuda")
    # zero the parameter gradients
    optimizer_where.zero_grad()
    optimizer_what.zero_grad()
    # forward + backward + optimize
    avg, alpha = where(inputs)
    outputs = what(avg)

    mx,_ = torch.max(alpha,1)
    entropy = np.mean(-np.log2(mx.cpu().detach().numpy()))
    # print("entropy of batch", entropy)

    loss = criterion(outputs, labels) + entropy

    # loss = criterion(outputs, labels)
    # print statistics
    running_loss += loss.item()
    loss.backward()
    optimizer_where.step()
    optimizer_what.step()

  running_loss,anls_data = calculate_attn_loss(train_loader,what,where,criterion)
  analysis_data.append(anls_data)
  print('epoch: [%d] loss: %.3f' %(epoch + 1,running_loss)) 
  loss_curi.append(running_loss)   #loss per epoch
  if running_loss<=0.06:
    break
print('Finished Training')
correct = 0
total = 0
with torch.no_grad():
  for data in train_loader:
    images, labels,_ = data
    images = images.double()
    images, labels = images.to("cuda"), labels.to("cuda")
    avg, alpha = where(images)
    outputs  = what(avg)
    _, predicted = torch.max(outputs.data, 1)
    total += labels.size(0)
    correct += (predicted == labels).sum().item()

print('Accuracy of the network on the 3000 train images: %d %%' % (  100 * correct / total))
    
analysis_data = np.array(analysis_data)
plt.figure(figsize=(6,6))
plt.plot(np.arange(0,epoch+2,1),analysis_data[:,0],label="ftpt")
plt.plot(np.arange(0,epoch+2,1),analysis_data[:,1],label="ffpt")
plt.plot(np.arange(0,epoch+2,1),analysis_data[:,2],label="ftpf")
plt.plot(np.arange(0,epoch+2,1),analysis_data[:,3],label="ffpf")

plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
plt.savefig("trends_synthetic_300_300.png",bbox_inches="tight")
plt.savefig("trends_synthetic_300_300.pdf",bbox_inches="tight")

analysis_data[-1,:2]/3000
running_loss,anls_data = calculate_attn_loss(train_loader,what,where,criterion)
print(running_loss, anls_data)
what.eval()
where.eval()
alphas = []
max_alpha =[]
alpha_ftpt=[]
alpha_ffpt=[]
alpha_ftpf=[]
alpha_ffpf=[]
argmax_more_than_half=0
argmax_less_than_half=0
cnt =0
with torch.no_grad():
  for i, data in enumerate(train_loader, 0):
    inputs, labels, fidx = data
    inputs = inputs.double()
    inputs, labels = inputs.to("cuda"),labels.to("cuda")
    avg, alphas = where(inputs)
    outputs = what(avg)
    _, predicted = torch.max(outputs.data, 1)
    batch = len(predicted)
    mx,_ = torch.max(alphas,1)
    max_alpha.append(mx.cpu().detach().numpy())
    for j in range (batch):
      cnt+=1
      focus = torch.argmax(alphas[j]).item()
      if alphas[j][focus] >= 0.5 :
        argmax_more_than_half += 1
      else:
        argmax_less_than_half += 1

      if (focus == fidx[j].item() and predicted[j].item() == labels[j].item()):
          alpha_ftpt.append(alphas[j][focus].item())
          # print(focus, fore_idx[j].item(), predicted[j].item() , labels[j].item() )

      elif (focus != fidx[j].item() and predicted[j].item() == labels[j].item()):
          alpha_ffpt.append(alphas[j][focus].item())

      elif (focus == fidx[j].item() and predicted[j].item() != labels[j].item()):
          alpha_ftpf.append(alphas[j][focus].item())

      elif (focus != fidx[j].item() and predicted[j].item() != labels[j].item()):
          alpha_ffpf.append(alphas[j][focus].item())

np.mean(-np.log2(mx.cpu().detach().numpy()))
a = np.array([0.8,0.9])
-np.log2(a)
np.mean(-np.log2(a))
max_alpha = np.concatenate(max_alpha,axis=0)
print(max_alpha.shape, cnt)
np.array(alpha_ftpt).size, np.array(alpha_ffpt).size, np.array(alpha_ftpf).size, np.array(alpha_ffpf).size
plt.figure(figsize=(6,6))
_,bins,_ = plt.hist(max_alpha,bins=50,color ="c")
plt.title("alpha values histogram")
plt.savefig("attention_model_2_hist")
plt.figure(figsize=(6,6))
_,bins,_ = plt.hist(np.array(alpha_ftpt),bins=50,color ="c")
plt.title("alpha values in ftpt")
plt.savefig("attention_model_2_hist")
plt.figure(figsize=(6,6))
_,bins,_ = plt.hist(np.array(alpha_ffpt),bins=50,color ="c")
plt.title("alpha values in ffpt")
plt.savefig("attention_model_2_hist")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16802610114192496, 'fraction_numerical': 0.01980890235376369, 'mean_word_length': 3.580042689434365, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '33853', 'license': 'MIT', 'n_tokens_mistral': 3312, 'n_tokens_neox': 3203, 'n_words': 533, 'path': '13_mosaic_sparsity_regularizer/old codes and plots/older/interpretable codes loss 1 by/Synthetic_elliptical_blobs_interpretable_100_300.ipynb', 'repo_name': 'lnpandey/DL_explore_synth_data', 'size': '8582'}