<jupyter_start><jupyter_text>MLP Baseline<jupyter_code>import dgl
import networkx as nx
import pickle
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import hypertunity as ht
from sklearn import preprocessing

with open('/gpfs_home/spate116/data/spate116/GCN/E116/data/data_embedding.pickle', 'rb') as f:
    data_embedding = pickle.load(f)
    
with open('/gpfs_home/spate116/data/spate116/GCN/E116/data/data_class1.pickle', 'rb') as f:
    data = pickle.load(f)
    
X = data_embedding
y = torch.tensor(list(map(lambda x: x[0], data.y)), dtype=torch.long)
model = nn.Sequential(
    nn.Linear(94, 1000),
    nn.LeakyReLU(),
    nn.Dropout(),
    nn.Linear(1000, 500),
    nn.LeakyReLU(),
    nn.Dropout(),
    nn.Linear(500, 100),
    nn.LeakyReLU(),
    nn.Dropout(),
    nn.Linear(100, 50),
    nn.LeakyReLU(),
    nn.Dropout(),
    nn.Linear(50, 2),
    nn.Sigmoid()
)
from tqdm import tqdm
from sklearn.metrics import roc_auc_score, roc_curve

device = torch.device('cuda')
def train_model(net, X, y, epochs, learning_rate, train_mask, test_mask):
    model = net.to(device)
    X = X.to(device)
    y = y.to(device)
    samples = len(X)

    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)
    losses_train = []
    losses_test = []
    best_auc = -1
    correct_pred = [y[i].item() for i in test_mask]
    
    weight_one = sum(y.cpu().numpy().tolist())/samples
    weight = torch.tensor([weight_one, 1-weight_one]).to(device)
    
    pbar = tqdm(range(epochs))
    for epoch in pbar:
        model.train()
        logits = model(X.float())

        loss = F.cross_entropy(logits[train_mask], y[train_mask], weight=weight)
        loss_test = F.cross_entropy(logits[test_mask], y[test_mask], weight=weight)
        losses_train.append(loss.item())
        losses_test.append(loss_test.item())

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        model.eval()
        pred = list(map(lambda x: np.argmax(x, axis = 0), torch.exp(F.log_softmax(logits, 1)).cpu().detach().numpy()))
        auc = roc_auc_score(correct_pred, [pred[i] for i in test_mask], average='weighted')
        best_auc = best_auc if best_auc > auc else auc

        pbar.set_description('Best Test AUC: %.4f | Train Loss: %.4f | Test Loss: %.4f' % (best_auc, loss.item(), loss_test.item()))
            
    return losses_train, losses_test, model
import random
random.seed(30)
idx = list(range(len(y)))
random.shuffle(idx)
train_mask = idx[:10000]
test_mask = idx[10000:]
def weight_reset(m):
    if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
        m.reset_parameters()

model.apply(weight_reset)

losses_train, losses_test, model = train_model(model, X, y, 250, 0.0005, train_mask, test_mask)<jupyter_output>Best Test AUC: 0.8608 | Train Loss: 0.4155 | Test Loss: 0.4539: 100%|██████████| 250/250 [00:09<00:00, 26.30it/s]
{'fraction_non_alphanumeric': 0.14611260053619302, 'fraction_numerical': 0.037198391420911525, 'mean_word_length': 3.509063444108761, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '369326', 'license': 'MIT', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1103, 'n_words': 227, 'path': 'notebooks/GCN/Baseline.ipynb', 'repo_name': 'shalinkpatel/GCN_Integration', 'size': '2984'}