<jupyter_start><jupyter_text>make numpy array for training<jupyter_code>import time
import tqdm
import numpy as np 
import matplotlib.pyplot as plt
import pandas as pd
import torch 
import torch.nn as nn 
import torch.nn.functional as F 
import torch.optim as optim 
import torch.utils.data as utils
import torch.nn.init as init
from torch.autograd import Variable
f_rnd = pd.read_hdf("/data/t3home000/spark/LHCOlympics_previous/LHC-Olympics/Code/Nsubjettiness_mjj.h5")
dt = f_rnd.values
index_list = [0,1,2,3,4,5,7,8,9,10,11,12]
for i in index_list:
    dt[:,i] = (dt[:,i]-np.mean(dt[:,i]))/np.std(dt[:,i])
idx = dt[:,15]
bkg_idx = np.where(idx==0)[0]
signal_idx = np.where(idx==1)[0]
X = dt[:,[0,1,2,3,4,5,7,8,9,10,11,12]]
X.shape
y = idx
y.shape<jupyter_output><empty_output><jupyter_text>Normalize Inputs<jupyter_code>from sklearn.model_selection import train_test_split
#X_train, X_test, y_train, y_test = train_test_split(
#    X, y, test_size=0.2, random_state=42)
#X_train, X_val, y_train, y_val = train_test_split(
#    X_train, y_train, test_size=0.2, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(
    X, y, test_size=0.2, random_state=42)

X_val, X_test, y_val, y_test = train_test_split(
    X_val, y_val, test_size=0.5, random_state=42)
print(X_train.shape, X_val.shape, X_test.shape)
class trainData(utils.Dataset):
    
    def __init__(self, X_data, y_data):
        self.X_data = X_data
        self.y_data = y_data
        
    def __getitem__(self, index):
        return self.X_data[index], self.y_data[index]
        
    def __len__ (self):
        return len(self.X_data)



## test data    
class testData(utils.Dataset):
    
    def __init__(self, X_data):
        self.X_data = X_data
        
    def __getitem__(self, index):
        return self.X_data[index]
        
    def __len__ (self):
        return len(self.X_data)
    


train_data = trainData(torch.FloatTensor(X_train.astype(np.float)),torch.FloatTensor(y_train.astype(np.float)))
val_data = trainData(torch.FloatTensor(X_val.astype(np.float)),torch.FloatTensor(y_val.astype(np.float)))
test_data = testData(torch.FloatTensor(X_test.astype(np.float)))<jupyter_output><empty_output><jupyter_text>Make Dataloader<jupyter_code>import torch
torch.cuda.is_available()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
torch.cuda.get_device_name(0)
BATCH_SIZE = 1000
N_EPOCHS = 80
PRINT_INTERVAL = 2000
NUM_WORKERS = 4
LR = 1e-4<jupyter_output><empty_output><jupyter_text>NN classifier<jupyter_code>class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        
        self.linear1 = nn.Linear(12,84)
        self.linear2 = nn.Linear(84,42)
        self.linear3 = nn.Linear(42,21)
        self.linear4 = nn.Linear(21,10)
        self.linear5 = nn.Linear(10,5)
        self.linear_out = nn.Linear(5,1)
        
        self.dropout = nn.Dropout(p=0.1)
        self.batchnorm1 = nn.BatchNorm1d(84)
        self.batchnorm2 = nn.BatchNorm1d(42)
        self.batchnorm3 = nn.BatchNorm1d(21)
        self.batchnorm4 = nn.BatchNorm1d(10)
        self.batchnorm5 = nn.BatchNorm1d(5)
        

    def forward(self, x):
        # Max pooling over a (2, 2) window
        x = self.batchnorm1(self.linear1(x))
        x = F.leaky_relu(x)
        x = self.dropout(x)
        x = self.batchnorm2(self.linear2(x))
        x = F.leaky_relu(x)
        x = self.dropout(x)
        x = self.batchnorm3(self.linear3(x))
        x = F.leaky_relu(x)
        x = self.dropout(x)
        x = self.batchnorm4(self.linear4(x))
        x = F.leaky_relu(x)
        x = self.dropout(x)
        x = self.batchnorm5(self.linear5(x))
        x = F.leaky_relu(x)
        x = self.linear_out(x)
        return x




model = Net()

print(model)
model.to(device)
# optimizer
lr = 1e-4
optimizer = optim.Adam(model.parameters(), lr=lr)
criterion = nn.BCEWithLogitsLoss()
device
train_loader = utils.DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)
val_loader = utils.DataLoader(dataset=val_data, batch_size=100)
test_loader = utils.DataLoader(dataset=test_data, batch_size=100)
def binary_acc(y_pred, y_test):
    y_pred_tag = torch.round(torch.sigmoid(y_pred))

    correct_results_sum = (y_pred_tag == y_test).sum().float()
    acc = correct_results_sum/y_test.shape[0]
    acc = torch.round(acc * 100)
    
    return acc
def train():
    # set the train mode
    model.train()

    # loss of the epoch
    epoch_loss = 0
    epoch_acc = 0

    for x, y in train_loader:
        # reshape the data into [batch_size, 784]
        x,y = x.to(device), y.to(device)

        # update the gradients to zero
        optimizer.zero_grad()

        # forward pass
        y_pred = model(x)

        # reconstruction loss
        #recon_loss = F.binary_cross_entropy(x_sample, x, size_average=False)

        # kl divergence loss
        #kl_loss = 0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1.0 - z_var)

        # total loss
        #loss = recon_loss + kl_loss

        #BCE loss
        loss = criterion(y_pred, y.unsqueeze(1))
        acc = binary_acc(y_pred, y.unsqueeze(1))
        
        # backward pass
        loss.backward()
        epoch_loss += loss.item()
        epoch_acc += acc.item()

        # update the weights
        optimizer.step()

    return epoch_loss

def test():
    # set the evaluation mode
    model.eval()

    # test loss for the data
    test_loss = 0

    # we don't need to track the gradients, since we are not updating the parameters during evaluation / testing
    with torch.no_grad():
        for x, y in val_loader:
            # reshape the data
            #x = x.view(-1, 28 * 28)
            x,y = x.to(device), y.to(device)
            # forward pass
            y_pred = model(x)

            loss = criterion(y_pred, y.unsqueeze(1))
            acc = binary_acc(y_pred, y.unsqueeze(1))
            # total loss
            
            test_loss += loss.item()

    return test_loss
best_test_loss = float('inf')

for e in range(100):

    train_loss = train()
    test_loss = test()

    train_loss /= len(train_data)
    test_loss /= len(val_data)

    print(f'Epoch {e}, Train Loss: {train_loss:.7f}, Test Loss: {test_loss:.7f}')

    if best_test_loss > test_loss:
        best_test_loss = test_loss
        patience_counter = 1
    else:
        patience_counter += 1

    if patience_counter > 3:
        print('patience limit reached')
        break
torch.save(model.state_dict(), "/data/t3home000/spark/QUASAR/weights/fully_connected_ref_12inputs.h5")
model.eval()
y_pred_list = np.array([],dtype=np.float)
with torch.no_grad():
    for batch_idx, X_batch in enumerate(test_loader):
        X_batch = X_batch.to(device)
        y_test_pred = model(X_batch)
        y_test_pred = torch.sigmoid(y_test_pred)
        y_pred_tag = torch.round(y_test_pred)
        y_pred_list = np.append(y_pred_list,y_test_pred.cpu().numpy())
        

truesig = np.where(y_test==1)
truebkg = np.where(y_test==0)
sigscore = y_pred_list[truesig]
bkgscore = y_pred_list[truebkg]
sigscore.shape
print(sigscore)
bkgscore.shape
print(y_pred_list.shape,y_test.shape)
print(y_pred_list[:20],y_test[:20])
def get_tpr_fpr(sigloss,bkgloss):
    bins = np.linspace(0,1,100001)
    tpr = []
    fpr = []
    for cut in bins:
        tpr.append(np.where(sigloss>cut)[0].shape[0]/len(sigloss))
        fpr.append(np.where(bkgloss>cut)[0].shape[0]/len(bkgloss))
    
    return tpr,fpr  
tpr, fpr = get_tpr_fpr(sigscore,bkgscore)
np.save('supervised_ref_tpr.npy',tpr)
np.save('supervised_ref_fpr.npy',fpr)
plt.plot(fpr,tpr)
max(0,1e-6)
def get_precision_recall(sigloss,bkgloss,aetype='bkg'):
    bins = np.linspace(0,1,100001)
    tpr = []
    fpr = []
    precision = []
    for cut in bins:
        if aetype == 'sig':
            tpr.append(np.where(sigloss<cut)[0].shape[0]/len(sigloss))
            precision.append((np.where(sigloss<cut)[0].shape[0])/(np.where(bkgloss<cut)[0].shape[0]+np.where(sigloss<cut)[0].shape[0]))
            
        if aetype == 'bkg':
            if (np.where(bkgloss>cut)[0].shape[0]+np.where(sigloss>cut)[0].shape[0]) != 0:
                tpr.append(np.where(sigloss>cut)[0].shape[0]/len(sigloss))
                precision.append((np.where(sigloss>cut)[0].shape[0])/(np.where(bkgloss>cut)[0].shape[0]+np.where(sigloss>cut)[0].shape[0]))
    return precision,tpr      
precision, recall = get_precision_recall(sigscore,bkgscore)
np.save('supervised_ref_precision.npy',precision)
np.save('supervised_ref_recall.npy',recall)
plt.plot(recall,precision)<jupyter_output><empty_output><jupyter_text>Now time to train a simple NN<jupyter_code>plt.hist(data_branches[b'vjet0_msd0'][y_pred_list>-40],1000)
data
f = uproot.update("data/JetHT_s.root")
data   = uproot.open("data/JetHT_s.root")["Tree"]
data.extend({"NN_score":y_pred_list})
import numpy as np
N, D_in, H, D_out = 64, 1000, 100, 10
x = np.random.randn(N, D_in)
y = np.random.randn(N, D_out)
print(x.shape)
# Randomly initialize weights
w1 = np.random.randn(D_in, H)
w2 = np.random.randn(H, D_out)
learning_rate = 1e-6
for i in range(500):
    #forward pass: compute predicted y
    h = x.dot(w1)
    h_relu = np.maximum(h,0)
    y_pred = h_relu.dot(w2)
    
    loss = np.square(y_pred-y).sum()
    print(t,loss)
    
    # back propagation
    grad_y_pred = 2.0 * (y_pred - y)
    grad_w2  = h_relu.T.dot(grad_y_pred)
    g<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13486842105263158, 'fraction_numerical': 0.027589134125636672, 'mean_word_length': 3.1593115622241834, 'pattern_counts': {'":': 1, '<': 25, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '445097', 'license': 'MIT', 'n_tokens_mistral': 3785, 'n_tokens_neox': 3640, 'n_words': 630, 'path': 'new_flows/full_connected_reference.ipynb', 'repo_name': 'SJ001/QUASAR', 'size': '9424'}