<jupyter_start><jupyter_text>Google colab requirements<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
import os
os.chdir('/content/drive/My Drive/Thesis/test')
os.getcwd()
!pip install torch==1.6.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
!pip install torch-scatter==latest+cu101 -f https://pytorch-geometric.com/whl/torch-1.6.0.html
!pip install torch-sparse==latest+cu101 -f https://pytorch-geometric.com/whl/torch-1.6.0.html
!pip install torch-cluster==latest+cu101 -f https://pytorch-geometric.com/whl/torch-1.6.0.html
!pip install torch-spline-conv==latest+cu101 -f https://pytorch-geometric.com/whl/torch-1.6.0.html
!pip install torch-geometric<jupyter_output><empty_output><jupyter_text>Hyperparameters<jupyter_code>hidden_channels = 256
beta = 0.001
dropout = 0.5
lr = 0.01
num_epochs = 500
do_tsne = False

# x_RBM
x_RBM_num_hid = 4096
x_RBM_num_epochs = 1000
x_RBM_lr = 0.001
x_RBM_num_CD = 1
x_RBM_batch_size= 64

# z0_RBM
z0_RBM_num_hid = 4096
z0_RBM_num_epochs = 1000
z0_RBM_lr = 0.001
z0_RBM_num_CD = 1
z0_RBM_batch_size= 64

# z1_RBM
z1_RBM_num_hid = 4096
z1_RBM_num_epochs = 1000
z1_RBM_lr = 0.001
z1_RBM_num_CD = 1
z1_RBM_batch_size= 64

# z2_RBM
z2_RBM_num_hid = 4096
z2_RBM_num_epochs = 1000
z2_RBM_lr = 0.001
z2_RBM_num_CD = 1
z2_RBM_batch_size= 64<jupyter_output><empty_output><jupyter_text>Libraries<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

import plotly.express as px
from bokeh.palettes import Category20_20, Category20b_20, Accent8

from tqdm import tqdm
import sys
sys.path.insert(0, "../..") # denoiseRBM
sys.path.insert(1, "..") # examples

import os
import os.path as osp

from denoiseRBM.distort_X import distort_x
from denoiseRBM.distort_A import distort_a
from denoiseRBM.RBM import B_RBM
from denoiseRBM.utils import Z, plot_tsne
from denoiseRBM.denoiseRBM import denoiseRBM
from ogbn_arxiv import ogbn_arxiv, test
from DNN import DNN, train
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
Z_ = Z(d=hidden_channels, beta=beta)

results = {
    # Corrupted data
    "X_c": {
        "phi": {},
        "psi_0": {},
        "psi_1": {},
        "psi_2": {}
    },
    # Blanked out data
    "X_z": {
        "phi": {},
        "psi_0": {},
        "psi_1": {},
        "psi_2": {}
    }
}<jupyter_output><empty_output><jupyter_text>Paths to store results<jupyter_code>chkpt_path = "MLP"
if not osp.exists(chkpt_path):
    os.mkdir(chkpt_path)<jupyter_output><empty_output><jupyter_text>Initialize ogbn-arxiv and MLP model<jupyter_code>x, y_true, edge_index, idx, nodes, x_distorted, A_distorted, evaluator = ogbn_arxiv("data")
MLP_model = DNN(NN=nn.Linear, in_channels=x.size(-1), hidden_channels=hidden_channels, 
                out_channels=len(np.unique(y_true.numpy())), device=device, mathcal_Z=None,
                dropout=0.5)
x_distorted<jupyter_output><empty_output><jupyter_text>Train MLP<jupyter_code>MLP_path = osp.join(chkpt_path, "MLP.pt")
if not osp.exists(MLP_path):
    MLP_model.reset_parameters()
    MLP_optimizer = Adam(MLP_model.parameters(), lr=lr)
    for epoch in range(num_epochs):
        loss = train(MLP_model, MLP_optimizer, F.nll_loss, x, y_true, idx["train"])
        acc = test(MLP_model, x, y_true, idx, evaluator)
        print(f"Epoch: {epoch:02d}, "
              f"Loss: {loss:.4f}, "
              f"Train: {100 * acc['train']:.2f}%, "
              f"Valid: {100 * acc['val']:.2f}%, "
              f"Test: {100 * acc['test']:.2f}%")
    torch.save(MLP_model.state_dict(), MLP_path)<jupyter_output><empty_output><jupyter_text>Load MLP model<jupyter_code>MLP_model.load_state_dict(torch.load(MLP_path, map_location=device))
z, _, _ = MLP_model(x)<jupyter_output><empty_output><jupyter_text>Verify that the accuracies from all the representations are the same Input layer<jupyter_code>x_acc = test(MLP_model, x, y_true, idx, evaluator)
print(f"Train: {100 * x_acc['train']:.2f}%, "
      f"Valid: {100 * x_acc['val']:.2f}%, "
      f"Test: {100 * x_acc['test']:.2f}%")
if do_tsne:
    x_train_fig = plot_tsne(x, idx["train"], 3, y_true[idx["train"]])
    x_train_fig.show()
if do_tsne:
    x_val_fig = plot_tsne(x, idx["val"], 3, y_true["val"])
    x_val_fig.show()
if do_tsne:
    x_test_fig = plot_tsne(x, idx["test"], 3, y_true["test"])
    x_test_fig.show()<jupyter_output><empty_output><jupyter_text>First hidden layer<jupyter_code>z0_acc = test(MLP_model, x, y_true, idx, evaluator, z[0], layer=1)
print(f"Train: {100 * z0_acc['train']:.2f}%, "
      f"Valid: {100 * z0_acc['val']:.2f}%, "
      f"Test: {100 * z0_acc['test']:.2f}%")
if do_tsne:
    z0_train_fig = plot_tsne(z[0], idx["train"], 3, y_true["train"])
    z0_train_fig.show()
if do_tsne:
    z0_val_fig = plot_tsne(z[0], idx["val"], 3, y_true["val"])
    z0_val_fig.show()
if do_tsne:
    z0_test_fig = plot_tsne(z[0], idx["test"], 3, y_true["test"])
    z0_test_fig.show()<jupyter_output><empty_output><jupyter_text>Second hidden layer<jupyter_code>z1_acc = test(MLP_model, x, y_true, idx, evaluator, z[1], layer=2)
print(f"Train: {100 * z1_acc['train']:.2f}%, "
      f"Valid: {100 * z1_acc['val']:.2f}%, "
      f"Test: {100 * z1_acc['test']:.2f}%")
if do_tsne:
    z1_train_fig = plot_tsne(z[1], idx["train"], 3, y_true["train"])
    z1_train_fig.show()
if do_tsne:
    z1_val_fig = plot_tsne(z[1], idx["val"], 3, y_true["val"])
    z1_val_fig.show()
if do_tsne:
    z1_test_fig = plot_tsne(z[1], idx["test"], 3, y_true["test"])
    z1_test_fig.show()<jupyter_output><empty_output><jupyter_text>Output layer<jupyter_code>z2_acc = test(MLP_model, x, y_true, idx, evaluator, z[2], layer=3)
print(f"Train: {100 * z2_acc['train']:.2f}%, "
      f"Valid: {100 * z2_acc['val']:.2f}%, "
      f"Test: {100 * z2_acc['test']:.2f}%")
if do_tsne:
    z2_train_fig = plot_tsne(z[2], idx["train"], 3, y_true["train"])
    z2_train_fig.show()
if do_tsne:
    z2_val_fig = plot_tsne(z[2], idx["val"], 3, y_true["val"])
    z2_val_fig.show()
if do_tsne:
    z2_test_fig = plot_tsne(z[2], idx["test"], 3, y_true["test"])
    z2_test_fig.show()<jupyter_output><empty_output><jupyter_text>Reconstruct representations Noisy node feature matrix<jupyter_code>x_distorted
for x_type in ["X_c", "X_z"]:
    for perc in range(0, 101, 10):
        reqd_x = x_distorted[x_type][perc].to(device)
        acc = test(MLP_model, reqd_x, y_true, idx, evaluator)
        print(f"{x_type}: n_X: {perc}%, Train: {100 * acc['train']:.4f}%, "
              f"Val: {100 * acc['val']:.4f}%, Test: {100 * acc['test']:.4f}%")
        z, _, _ = MLP_model(reqd_x)
        for split in acc: results[x_type]["phi"][split][perc] = [acc[split], 
                                                                 reqd_x[split], 
                                                                 z[0][split], 
                                                                 z[1][split], 
                                                                 z[2][split]
                                                                ]<jupyter_output><empty_output><jupyter_text>Reconstructions of noisy node feature matrix<jupyter_code>x_rbm = B_RBM("Gaussian", x.size(1), x_RBM_num_hid).to(device)
psi_0_res = denoiseRBM(MLP_model, x_rbm, x, x_distorted, idx, device, chkpt_path, x_RBM_num_hid, 
                       x_RBM_num_epochs, x_RBM_lr, x_RBM_CD, x_RBM_batch_size, 10)
for x_type in ["X_c", "X_z"]:
    results[x_type]["psi_0"] = psi_0_res[x_type]<jupyter_output><empty_output><jupyter_text>Reconstructions of first hidden layer representations<jupyter_code>z0_rbm = B_RBM("Gaussian", z[0].size(1), z0_RBM_num_hid).to(device)
psi_1_res = denoiseRBM(MLP_model, z0_rbm, z[0], x_distorted, idx, device, chkpt_path, 
                       x_RBM_num_hid, x_RBM_num_epochs, x_RBM_lr, x_RBM_CD, x_RBM_batch_size, 
                       10, layer=1)
for x_type in ["X_c", "X_z"]:
    results[x_type]["psi_1"] = psi_1_res[x_type]<jupyter_output><empty_output><jupyter_text>Reconstructions of second hidden layer representations<jupyter_code>z1_rbm = B_RBM("Gaussian", z[1].size(1), z1_RBM_num_hid).to(device)
psi_2_res = denoiseRBM(MLP_model, z1_rbm, z[1], x_distorted, idx, device, chkpt_path, 
                       x_RBM_num_hid, x_RBM_num_epochs, x_RBM_lr, x_RBM_CD, x_RBM_batch_size, 
                       10, layer=2)
for x_type in ["X_c", "X_z"]:
    results[x_type]["psi_2"] = psi_2_res[x_type]<jupyter_output><empty_output><jupyter_text>Reconstructions of third hidden layer representations<jupyter_code>z2_rbm = B_RBM("Gaussian", z[2].size(1), z2_RBM_num_hid).to(device)
psi_3_res = denoiseRBM(MLP_model, z2_rbm, z[2], x_distorted, idx, device, chkpt_path, 
                       x_RBM_num_hid, x_RBM_num_epochs, x_RBM_lr, x_RBM_CD, x_RBM_batch_size, 
                       10, layer=3)
for x_type in ["X_c", "X_z"]:
    results[x_type]["psi_3"] = psi_3_res[x_type]
for x_type in ["X_c", "X_z"]:
    for split in ["train", "val", "test"]:
        for perc in range(0, 101, 10):
            np.savez(osp.join(chkpt, f"{x_type}_{perc}_A_c_0_{split}.npz"),
                     phi_acc = results[x_type]["phi"][split][perc][0],
                     phi_z0 = results[x_type]["phi"][split][perc][1],
                     phi_z1 = results[x_type]["phi"][split][perc][2],
                     phi_z2 = results[x_type]["phi"][split][perc][3],
                     phi_z3 = results[x_type]["phi"][split][perc][4],

                     psi_0_acc = results[x_type]["psi_0"][split][perc][0],
                     psi_0_z0 = results[x_type]["psi_0"][split][perc][1],
                     psi_0_z1 = results[x_type]["psi_0"][split][perc][2],
                     psi_0_z2 = results[x_type]["psi_0"][split][perc][3],
                     psi_0_z3 = results[x_type]["psi_0"][split][perc][4],

                     psi_1_acc = results[x_type]["psi_1"][split][perc][0],
                     psi_1_z1 = results[x_type]["psi_1"][split][perc][1],
                     psi_1_z2 = results[x_type]["psi_1"][split][perc][2],
                     psi_1_z3 = results[x_type]["psi_1"][split][perc][3],

                     psi_2_acc = results[x_type]["psi_2"][split][perc][0],
                     psi_2_z2 = results[x_type]["psi_2"][split][perc][1],
                     psi_2_z3 = results[x_type]["psi_2"][split][perc][2],

                     psi_3_acc = results[x_type]["psi_3"][split][perc][0],
                     psi_3_z3 = results[x_type]["psi_3"][split][perc][1]
                    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15475181312988603, 'fraction_numerical': 0.03711029481021004, 'mean_word_length': 3.459470810583788, 'pattern_counts': {'":': 10, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '78631', 'license': 'MIT', 'n_tokens_mistral': 4566, 'n_tokens_neox': 4357, 'n_words': 680, 'path': 'examples/ogbn-arxiv/denoise_ogbn_arxiv_MLP.ipynb', 'repo_name': 'ankithmo/denoiseRBM', 'size': '10617'}