<jupyter_start><jupyter_text>MeSH Dataset Class...<jupyter_code>class MESHDataset(Dataset):
    def __init__(self,numpy_file,label_file):
     
      try:
        self.data= np.load(numpy_file)
        self.labels = np.load(label_file)
      except Exception as err:
        raise Exception(f'ERROR OPENING FILES: {numpy_file} | {label_file}. See Error below. \n {err}')  
       
    def __len__(self):
        return self.data.shape[0]
    def __getitem__(self, idx):
        #Get the element with `idx`
        #Output an 89*89 matrix
        return self.data[idx].flatten(), self.labels[idx]<jupyter_output><empty_output><jupyter_text>Baseline Model...<jupyter_code>class BaselineModel(nn.Module):

    def __init__(self, matrix_size=89):
        super(BaselineModel, self).__init__()
        self.linear1 = nn.Linear(matrix_size**2, (matrix_size**2)//2)
        self.linear2 = nn.Linear((matrix_size**2)//2, matrix_size**2//4)
        self.linear3 = nn.Linear((matrix_size**2)//4, NUM_CLASSES)
        
    def forward(self, x):

        x = self.linear1(x)
        x = F.relu(x) 

        x = self.linear2(x)
        x = F.relu(x)

        x = self.linear3(x)
        return x<jupyter_output><empty_output><jupyter_text>Creating the dataloaders...<jupyter_code>train_dataset = MESHDataset(os.path.join(DATA_DIR,'train.npy'), os.path.join(DATA_DIR,'grouped_train_labels.npy'))
dev_dataset = MESHDataset(os.path.join(DATA_DIR,'dev.npy'), os.path.join(DATA_DIR,'grouped_dev_labels.npy'))
test_dataset = MESHDataset(os.path.join(DATA_DIR,'test.npy'),os.path.join(DATA_DIR,'test_labels.npy'))


train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
dev_dataloader = DataLoader(dev_dataset, batch_size=BATCH_SIZE, shuffle=True)
test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)<jupyter_output><empty_output><jupyter_text>Creating the model...<jupyter_code>model = BaselineModel().to(device)
model<jupyter_output><empty_output><jupyter_text>Training Configuration...<jupyter_code>from sklearn.metrics import accuracy_score
def compute_accuracy(pred, target):
  return float(100 * accuracy_score(target.detach().numpy(), pred.argmax(-1).detach().numpy()))
#Define criterion: Categorical cross entropy
criterion = nn.CrossEntropyLoss()
#Define optimizer. For now, Adam
optimizer = optim.Adam(model.parameters(), lr=0.0001)

PRINT_FREQ = 15
DO_VALIDATION_STEP=15
NUM_EPOCHS = 1500
PATIENCE=50

PATH=os.path.join(DATA_DIR,'best_model_3')
best_acc = 0

liveloss = PlotLosses()
patience=0
for epoch in range(NUM_EPOCHS):
  for i,(features, label) in enumerate(train_dataloader):
    features, label = features.to(device), label.to(device)
    optimizer.zero_grad()
    pred = model(features.float())
    loss = criterion(pred,label)
    
    loss.backward()
    optimizer.step()

    if i%DO_VALIDATION_STEP==0:
      #Do validation
      model.eval()
      val_losses=[]
      val_acc_list=[]
      for val_features, val_label in dev_dataloader:
         val_features, val_label = val_features.to(device), val_label.to(device)
         val_pred = model(val_features.float())
         val_loss = criterion(val_pred,val_label)
         val_losses.append(val_loss.item())
         val_acc_list.append(compute_accuracy(val_pred.cpu(),val_label.cpu()))

      val_loss_ = np.mean(val_losses) 
      val_acc = np.mean(val_acc_list)
      if val_acc> best_acc:
        best_acc=val_acc
        patience=0
        #Save model weights
        torch.save(model.state_dict(), PATH)
      else:
        patience+=1
      if patience >= PATIENCE:
        break
      logs={'loss':loss.item(),'val_loss':val_loss_,'val_accuracy':val_acc}
      liveloss.update(logs)
      liveloss.send()  
      model.train()<jupyter_output><empty_output><jupyter_text>Compute test metrics...<jupyter_code>model.eval()
test_acc_list=[]
#f1_list=[]
for test_features, test_label in test_dataloader:
  test_features, test_label =test_features.to(device), test_label.to(device)
  test_pred = model(test_features.float())

  test_acc_list.append(compute_accuracy(test_pred.cpu(),test_label.cpu()))
  #f1_list.append(f1_loss(test_label, test_pred).item())

test_acc = np.mean(test_acc_list)
#test_f1 = np.mean(f1_list)

print(f'Test Metrics \n _________________ \n Mean Accuracy: {test_acc} ')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13130615065653076, 'fraction_numerical': 0.0119788067265607, 'mean_word_length': 4.263030303030303, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '25743', 'license': 'MIT', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1556, 'n_words': 235, 'path': 'models/NeuralNets/TrainingNotebooks/FC_MeSH_Model_5.ipynb', 'repo_name': 'SisonkeBiotik-Africa/MeSH2Matrix', 'size': '4341'}