<jupyter_start><jupyter_text>DescriptionDiffPool with Louvain based community Install Packages<jupyter_code>!pip install utils
!pip install pyhocon
!pip install scikit-learn-extra
!pip install tensorboardX
!pip install dgl==0.4<jupyter_output>Collecting utils
  Downloading https://files.pythonhosted.org/packages/55/e6/c2d2b2703e7debc8b501caae0e6f7ead148fd0faa3c8131292a599930029/utils-1.0.1-py2.py3-none-any.whl
Installing collected packages: utils
Successfully installed utils-1.0.1
Collecting pyhocon
[?25l  Downloading https://files.pythonhosted.org/packages/af/c4/39cfc164ed77ddfe7dd80ff31b065c2b18583c4ee0a4336979c86c4b5d8f/pyhocon-0.3.57.tar.gz (110kB)
[K     |████████████████████████████████| 112kB 5.1MB/s 
[?25hRequirement already satisfied: pyparsing>=2.0.3 in /usr/local/lib/python3.6/dist-packages (from pyhocon) (2.4.7)
Building wheels for collected packages: pyhocon
  Building wheel for pyhocon (setup.py) ... [?25l[?25hdone
  Created wheel for pyhocon: filename=pyhocon-0.3.57-cp36-none-any.whl size=18540 sha256=fb059b2e7cd50929ed16fe178b3e3170cd0665c12d135b56778ff324f517e77b
  Stored in directory: /root/.cache/pip/wheels/43/ae/39/250b80e3aa8082c5fbb3f14782b9cad47de0b885f22e927a8f
Successfully built pyhocon
Install[...]<jupyter_text>Import Packages<jupyter_code># Default

import os
import re
import sys
import math
import random
import pyhocon
import argparse
import itertools
import seaborn
import atexit
import shutil
from tqdm import tqdm
import utils
import time
import matplotlib.pyplot as plt

import matplotlib.pyplot as plt
from collections import defaultdict

from __future__ import division
from __future__ import print_function
# from tensorboardX import SummaryWriter

import dgl
from dgl.data.utils import download, extract_archive, get_download_dir, _get_dgl_url


# PyTorch

import torch
import numpy as np
import scipy
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import torch.utils.data
from torch.utils.data import Dataset,DataLoader, random_split

# Sklearn

from sklearn.utils import shuffle
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score

# Utility
import community as community_louvain
import networkx as nx
from sklearn.manifold import TSNE
import seaborn as sns
import matplotlib.patheffects as PathEffects
import time<jupyter_output><empty_output><jupyter_text>Utility  Import GIB Utility<jupyter_code>sys.path.append(os.path.abspath('/content/drive/MyDrive/Graph_Pooling'))
from GIB import *<jupyter_output><empty_output><jupyter_text>Gaussian XIB<jupyter_code>from torch.distributions.normal import Normal
def gaussian_xib(embs, s_l, dim, device):
  assert embs.shape[1] == dim
  dim = int(dim/2)
  prior = Normal(loc=torch.zeros(s_l.shape[1], dim).to(device),
                 scale=torch.ones(s_l.shape[1], dim).to(device),)
  posterior, _ = reparameterize(model=None, input=embs, mode='diag', size=dim)

  # print(posterior.shape, s_l.shape)
  # n1, n2 = torch.nonzero(s_l, as_tuple=True)
  # posterior = posterior[n2]
  # print(posterior.shape, s_l.shape)
  # print(prior.shape)
  # print('n1: {} n2: {}'.format(n1, n2))
  xib = torch.distributions.kl.kl_divergence(posterior, prior).sum(-1).unsqueeze(0).mean(1).unsqueeze(0)
  embs = embs[:, :dim]
  # print(xib.shape, embs.shape)
  return embs, xib<jupyter_output><empty_output><jupyter_text>Feature Perturbation<jupyter_code>def feature_perturb(x, perturb_ratio):
  x_max_mean = 0
  x_c = []
  if perturb_ratio != 0.0:
    for tx in x:
      # print(tx)
      # print(type(tx), tx.shape)
      # print(tx.max(1)[0].shape)
      tx_c = tx + torch.randn(tx.shape) * tx.max(1)[0].mean() * perturb_ratio
      # x + torch.randn(x.shape) * x_max_mean * perturb_ratio
      x_c.append(tx_c)
  else:
    x_c = deepcopy(x)
  
  return x_c<jupyter_output><empty_output><jupyter_text>Data Loader<jupyter_code>class TUDataset(Dataset):
    _url = r"https://ls11-www.cs.uni-dortmund.de/people/morris/graphkerneldatasets/{}.zip"

    def __init__(self, name):

        self.name = name

        # self.extract_dir = self._download()
        self.extract_dir = '/content/drive/My Drive/Graph_Pooling'
#         self.extract_dir = os.getcwd() + '/data'
        DS_edge_list = self._idx_from_zero(
            np.loadtxt(self._file_path("A"), delimiter=",", dtype=int))
        DS_indicator = self._idx_from_zero(
            np.loadtxt(self._file_path("graph_indicator"), dtype=int))
        DS_graph_labels = self._idx_from_zero(
            np.loadtxt(self._file_path("graph_labels"), dtype=int))

        g = dgl.DGLGraph()
        g.add_nodes(int(DS_edge_list.max()) + 1)
        g.add_edges(DS_edge_list[:, 0], DS_edge_list[:, 1])

        node_idx_list = []
        for idx in range(np.max(DS_indicator) + 1):
            node_idx = np.where(DS_indicator == idx)
            node_idx_list.append(node_idx[0])
        self.graph_lists = g.subgraphs(node_idx_list)
        self.graph_labels = DS_graph_labels

        try:
            DS_node_labels = self.to_onehot(
                self._idx_from_zero(np.loadtxt(self._file_path("node_labels"), dtype=int)))
            for idxs, g in zip(node_idx_list, self.graph_lists):
                g.ndata['feat'] = DS_node_labels[idxs, :]
        except IOError:
            print("No Node Label Data")

        try:
            DS_node_attr = np.loadtxt(self._file_path("node_attributes"), delimiter=",")
            shape2 = DS_node_attr.shape[1] if len(DS_node_attr.shape)>1 else 1
            DS_node_attr = DS_node_attr.reshape(DS_node_attr.shape[0],shape2)
            for idxs, g in zip(node_idx_list, self.graph_lists):
                g.ndata['feat'] = DS_node_attr[idxs,:]
        except IOError:
            print("No Node Attribute Data")

    def __getitem__(self, idx):
        g = self.graph_lists[idx]
        return (g.adjacency_matrix().to_dense(), g.ndata['feat']), self.graph_labels[idx]

    def __len__(self):
        return len(self.graph_lists)

    def _download(self):
        download_dir = get_download_dir()
        zip_file_path = os.path.join(download_dir, "tu_{}.zip".format(self.name))
        download(self._url.format(self.name), path=zip_file_path)
        extract_dir = os.path.join(download_dir, "tu_{}".format(self.name))
        extract_archive(zip_file_path, extract_dir)
        return extract_dir

    def _file_path(self, category):
        return os.path.join(self.extract_dir, self.name, "{}_{}.txt".format(self.name, category))

    @staticmethod
    def _idx_from_zero(idx_tensor):
        return idx_tensor - np.min(idx_tensor)

    @staticmethod
    def to_onehot(label_tensor):
        label_num = label_tensor.shape[0]
        assert np.min(label_tensor) == 0
        one_hot_tensor = np.zeros((label_num, np.max(label_tensor) + 1))
        one_hot_tensor[np.arange(label_num), label_tensor] = 1
        return one_hot_tensor


class CollateFn:
    def __init__(self, device='cpu'):
        self.device = device

    def __call__(self, batch):
        adj_tensor_list = []
        features_list = []
        mask_list = []
        node_list = []
        # (adj, features), labels = list(zip(*batch))
        max_num_nodes = max([g[0][0].shape[0] for g in batch])
        labels = []
        for (A, F), L in batch:
            adj_tensor_list.append(A.float())
            features_list.append(F.float())
            labels.append(L)
            # labels.append(L)
            # length = A.shape[0]
            # pad_len = max_num_nodes - length  #up, down,   left, right
            # adj_tensor_list.append(np.pad(A, ((0, pad_len), (0, pad_len)), mode='constant'))
            # features_list.append(np.pad(F, ((0, pad_len), (0, 0)), mode='constant'))
            # node_list.append(length)
            # mask = np.zeros(max_num_nodes)
            # mask[:length] = 1
            # mask_list.append(mask)
        # return torch.from_numpy(np.stack(adj_tensor_list, 0)).float().to(self.device), \
        #        torch.from_numpy(np.stack(features_list, 0)).float().to(self.device), \
        #        torch.from_numpy(np.stack(mask_list, 0)).float().to(self.device), \
        #        torch.from_numpy(np.stack(labels, 0)).long().to(self.device), \
        #        torch.from_numpy(np.stack(node_list, 0)).long().to(self.device)
        return adj_tensor_list,features_list,\
               torch.from_numpy(np.stack(labels, 0)).long().to(self.device)<jupyter_output><empty_output><jupyter_text>Models<jupyter_code>class BatchedGraphSAGE(nn.Module):
    def __init__(self, infeat, outfeat, device='cpu', use_bn=True, mean=False, add_self=False):
        super().__init__()
        self.add_self = add_self
        self.use_bn = use_bn
        self.device = device
        self.mean = mean
        self.W = nn.Linear(2*infeat, outfeat, bias=True)
        # nn.init.xavier_uniform_(self.W.weight, gain=nn.init.calculate_gain('relu'))

    def forward(self, x, adj, mask=None):
        # print(type(adj), type(x))
        if self.add_self:
            adj = adj 
            # + torch.eye(adj.size(0)).to(self.device)

        if self.mean:
            adj = adj / adj.sum(1, keepdim=True)

        h_k_N = torch.matmul(adj, x)
        h_k = self.W(torch.cat([x,h_k_N], dim=1))
        h_k = F.normalize(h_k, dim=1, p=2)
        h_k = F.relu(h_k)
        if self.use_bn and h_k.shape[0]>1:
            self.bn = nn.BatchNorm1d(h_k.size(0)).to(self.device)
            h_k = self.bn(h_k)
        if mask is not None:
            h_k = h_k * mask.unsqueeze(2).expand_as(h_k)
        return h_k


class BatchedDiffPool(nn.Module):
    def __init__(self, nfeat, nhid, is_final=False, device='cpu', link_pred=False):
        super(BatchedDiffPool, self).__init__()
        self.link_pred = link_pred
        self.device = device
        self.is_final = is_final
        double_nhid = nhid*2 
        self.mlp1 = nn.Linear(nfeat, nhid, bias=True)
        self.mlp2 = nn.Linear(nhid, double_nhid, bias=True)
        self.agg_param = nn.Parameter(torch.FloatTensor(12*double_nhid,double_nhid))
        self.mlp3 = nn.Linear(double_nhid, double_nhid, bias=True)
        self.mlp4 = nn.Linear(double_nhid, double_nhid, bias=True)
        self.MPN = BatchedGraphSAGE(nhid, nhid, device=self.device, use_bn=False)
        self.log = {}
        self.link_pred_loss = 0
        self.entropy_loss = 0

        self.out_channels = nhid
        self.nhid = double_nhid
        self.init_params()
        # self.device = device
      
    
    def init_params(self):
      for param in self.parameters():
        if len(param.size()) == 2:
          nn.init.xavier_uniform_(param,gain=nn.init.calculate_gain('relu'))


    
    def get_assign_mat(self,adj):
      
      
      n1 = adj.shape[0]

      # rows, cols = np.where(adj == 1)
      rows, cols = torch.nonzero(adj, as_tuple = True)
      edges = zip(rows.tolist(), cols.tolist())
      
      G = nx.Graph()
      for u in range(n1):
        G.add_node(u)
      G.add_edges_from(edges)
      cluster_coeff = nx.clustering(G)
      deg_centrality = nx.degree_centrality(G)
      

      dendo = community_louvain.generate_dendrogram(G)
      assign_mat_list = []
      cc_mat_list = []
      deg_mat_list = []
      # print(len(dendrogram))
      # node2com = community_louvain.best_partition(G)

      for level in range(len(dendo)-1,max(-1,len(dendo)-3),-1):
        node2com = community_louvain.partition_at_level(dendo, level)
        
        com2node = dict()
        for node in node2com:
          if node2com[node] not in com2node:
            com2node[node2com[node]] = list()
          com2node[node2com[node]].append(node)
        
        n2 = len(com2node)
        assign_mat = torch.zeros(n1,n2).float().to(self.device)
        
        for u in com2node:
          for v in com2node[u]:
            assign_mat[v][u] = 1
        
        cluster_coeff = torch.exp(torch.FloatTensor([cluster_coeff[u] for u in range(n1)])).reshape(n1, 1).to(self.device)
        deg_centrality = torch.exp(torch.FloatTensor([deg_centrality[u] for u in range(n1)])).reshape(n1, 1).to(self.device)
        
        cls_sum = torch.mm(torch.transpose(assign_mat, -1, 0), cluster_coeff)
        deg_sum = torch.mm(torch.transpose(assign_mat, -1, 0), deg_centrality)

        cls_sum = torch.mm(assign_mat, cls_sum)
        deg_sum = torch.mm(assign_mat, deg_sum)

        cc_mat = cluster_coeff*(1/cls_sum)
        deg_mat = deg_centrality*(1/deg_sum)
        
        assign_mat_list.append(assign_mat)
        cc_mat_list.append(cc_mat)
        deg_mat_list.append(deg_mat)

      return assign_mat_list,cc_mat_list,deg_mat_list

    def aggregator(self,z_l,s_l):
      n1,n2 = s_l.shape[0],s_l.shape[1]
      sum_feat = torch.matmul(s_l.transpose(-1, -2), z_l).to(self.device)
      col_sum = torch.sum(s_l,dim=0).reshape(s_l.shape[1],1).to(self.device)
      col_sum = torch.where(col_sum>0,col_sum,torch.ones(col_sum.shape, device=self.device))
      mean_feat = sum_feat/col_sum

      max_feat = []
      
      for j in range(n2):
        temp = torch.zeros(1,self.nhid, device=self.device)
        for i in range(n1):
          if s_l[i][j] == 1:
            cur_feat = z_l[i].reshape(1,self.nhid)
            con = torch.cat([temp,cur_feat],dim=0)
            temp = torch.max(con,dim=0).values.reshape(1,self.nhid)
        max_feat.append(temp)
      max_feat = torch.cat(max_feat,dim=0)

      min_feat = []
      n1,n2 = s_l.shape[0],s_l.shape[1]
      for j in range(n2):
        temp = []
        for i in range(n1):
          if s_l[i][j] == 1:
            cur_feat = z_l[i].reshape(1,self.nhid)
            if len(temp) == 0:
              temp = cur_feat
            else:
              con = torch.cat([temp,cur_feat],dim=0)
              temp = torch.min(con,dim=0).values.reshape(1,self.nhid)
        if len(temp)== 0:
          temp = torch.zeros(1,self.nhid)
        min_feat.append(temp)
      min_feat = torch.cat(min_feat,dim=0)
      return sum_feat, mean_feat, max_feat, min_feat

      
    def run_Louvain(self,x,adj):
      b_sz = len(x)
      batch_gxib = []
      final_feat = []
      for i in range(b_sz):
        feat = x[i].to(self.device)
        g = adj[i].to(self.device)
        
        feat = self.mlp1(feat)
        feat = self.MPN(feat, g)
        feat = F.relu(feat)
        feat = self.mlp2(feat)
        feat = F.relu(feat)
        
        feat_original = feat
        
        assign_mat_list,cc_mat_list,deg_mat_list = self.get_assign_mat(g)
        graph_feat = torch.zeros(1,self.out_channels).float().to(self.device)
        for i in range(len(assign_mat_list)):

            s_l = assign_mat_list[i]
            cc_mat = cc_mat_list[i]
            deg_mat = deg_mat_list[i]

            n1 = s_l.shape[0]
            n2 = s_l.shape[1]

            z_l = feat_original
            feat = []
            sum, mean,mx,mn = self.aggregator(z_l,s_l)
            feat.append(sum)
            feat.append(mean)
            feat.append(mx)
            feat.append(mn)
            # sum, mean,mx,mn = self.aggregator(cc_mat.mm(z_l),s_l)
            sum, mean,mx,mn = self.aggregator(cc_mat*z_l,s_l)
            feat.append(sum)
            feat.append(mean)
            feat.append(mx)
            feat.append(mn)
            # sum, mean, mx, mn = self.aggregator(deg_mat.mm(z_l),s_l)
            sum, mean, mx, mn = self.aggregator(deg_mat*z_l,s_l)
            feat.append(sum)
            feat.append(mean)
            feat.append(mx)
            feat.append(mn)

            feat = torch.cat(feat,dim=0).reshape(n2,self.nhid*12)
            feat = feat.mm(self.agg_param)


            feat = self.mlp3(feat)
            feat = F.relu(feat)
            feat = self.mlp4(feat)
            feat = F.relu(feat)

            # h1, h2 = torch.nonzero(s_l, as_tuple=True)
            # ext_embs = feat[h2]
            _, gxib = gaussian_xib(feat, s_l, feat.shape[1], self.device)
            batch_gxib.append(gxib)

            feat = feat[:,:self.out_channels]

            feat = torch.sum(feat,dim=0).reshape(1,self.out_channels)
            graph_feat += feat
            
        graph_feat/=len(assign_mat_list)
        final_feat.append(graph_feat)

      final_feat = torch.cat(final_feat,dim=0)
      return final_feat, batch_gxib       


    def forward(self, x, adj, mask=None, log=False):
        final_embd, batch_gxib = self.run_Louvain(x,adj)
        return final_embd, batch_gxib


class Classifier(nn.Module):
    def __init__(self, n_classes):
        super().__init__()
        self.classifier = nn.Sequential(nn.Linear(30, 50),
                                        nn.ReLU(),
                                        nn.Linear(50, n_classes))

    def forward(self, x):
        return self.classifier(x)


class BatchedModel(nn.Module):
    def __init__(self, pool_size, device, input_shape, n_classes, link_pred=False):
        super().__init__()
        self.input_shape = input_shape
        self.link_pred = link_pred
        self.device = device
        self.layers = nn.ModuleList([
            # BatchedGraphSAGE(input_shape, 30, device=self.device)
            # BatchedGraphSAGE(30, 30, device=self.device),
            BatchedDiffPool(input_shape, 30, device=self.device, link_pred=link_pred)#,
            # BatchedGraphSAGE(30, 30, device=self.device),
            # BatchedGraphSAGE(30, 30, device=self.device),
            # BatchedDiffPool(30, 1, 30, is_final=True, device=self.device)
        ])
        self.classifier = Classifier(n_classes)
        # writer.add_text(str(vars(self)))
        self.init_params()
    
    def init_params(self):
      for param in self.parameters():
        if len(param.size()) == 2:
          nn.init.xavier_uniform_(param,gain=nn.init.calculate_gain('relu'))

    def forward(self, x, adj):
        for layer in self.layers:
          
          x = layer(x, adj)

        # x = x * mask
        # readout_x = x.sum(dim=1)
        readout_x = x
        return readout_x

    def loss(self, output, labels):
        criterion = nn.CrossEntropyLoss()
        loss = criterion(output, labels)
        # if self.link_pred:
        #     for layer in self.layers:
        #         if isinstance(layer, BatchedDiffPool):
        #             loss = loss + layer.link_pred_loss.mean() + layer.entropy_loss.mean()

        return loss<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>def main():
#     utils.writer = SummaryWriter()

    parser = argparse.ArgumentParser()
    parser.add_argument('-f')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='Disables CUDA training.')
    parser.add_argument('--epochs', type=int, default=1000,
                        help='Number of epochs to train.')
    parser.add_argument('--link-pred', action='store_true', default=False,
                        help='Enable Link Prediction Loss')
    parser.add_argument('--dataset', default='ENZYMES', help="Choose dataset: ENZYMES, DD, NCI1")
    parser.add_argument('--batch-size', default=8, type=int, help="Choose dataset: ENZYMES, DD")
    parser.add_argument('--train-ratio', default=0.8, type=float, help="Train/Val split ratio")
    parser.add_argument('--pool-ratio', default=0.25, type=float, help="Train/Val split ratio")
    parser.add_argument('--beta', default=0.01, type=float, help="IB beta")
    parser.add_argument('--cuda', type=bool, default=False)
    parser.add_argument('--load_run', type=bool, default=False)
    parser.add_argument('--save_model', type=bool, default=True)
    
    PATH = '/content/drive/My Drive/Graph_Pooling'

    

	  
    
    title = 'Test F1,' \
            'Test F1(0.5),' \
            'Test F1(1.0),' \
            'Test F1(1.5),' \
            'Test F1(2.0),' \
            '\n'

    args = parser.parse_args()
#     utils.writer.add_text("args", str(args))
    device = "cuda" if not args.no_cuda and torch.cuda.is_available() else "cpu"
    # random.seed(args.seed)
	# np.random.seed(args.seed)
	# torch.manual_seed(args.seed)
	# torch.cuda.manual_seed_all(args.seed)

    file_name = PATH + '/' + args.dataset + '_summary' + '.csv'

    dataset = TUDataset(args.dataset)
    # dataset = MNIST(root="~/.torch/data/", transform=GraphTransform(device), download=True)
    dataset_size = len(dataset)
    train_size = int(dataset_size * args.train_ratio)
    test_size = int((dataset_size - train_size)/2)
    val_size = dataset_size - (train_size + test_size)
    max_num_nodes = max([item[0][0].shape[0] for item in dataset])
    n_classes = int(max([item[1] for item in dataset])) + 1
    train_data, val_data, test_data = random_split(dataset, (train_size, val_size, test_size))
    input_shape = int(dataset[0][0][1].shape[-1])
    train_loader = DataLoader(train_data, batch_size=args.batch_size, shuffle=True,
                              collate_fn=CollateFn(device))
    test_loader = DataLoader(test_data, batch_size=args.batch_size, shuffle=True,
                             collate_fn=CollateFn(device))
    val_loader = DataLoader(val_data, batch_size=args.batch_size, shuffle=True,
                             collate_fn=CollateFn(device))

    model = BatchedModel(pool_size=int(max_num_nodes * args.pool_ratio), device=device,
                         link_pred=args.link_pred, input_shape=input_shape, n_classes=n_classes).to(
        device)
    model.train()
    optimizer = optim.Adam(model.parameters())

    train_iter = 0
    epoch_losses_list = []
    ep_vt = []
    test_iter = 0
    mx_acc = [0,0,0,0,0]
    version = 0
    for e in range(args.epochs):
        print("Epoch: ",e)
        utils.e = e
        
        true_sample = 0
        model.train()
        st = time.time()
        for i, (adj, features, batch_labels) in enumerate(train_loader):

            train_iter += 1
            graph_feat, batch_gxib = model(features, adj)
            output = model.classifier(graph_feat)
            loss = model.loss(output, batch_labels)

            # print(batch_gxib)
            # print(torch.cat(batch_gxib, dim=0).shape)
            gxib_loss = torch.cat(batch_gxib, dim=0).mean()

            loss = loss + args.beta*gxib_loss 

            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 2.0)
            optimizer.step()
            optimizer.zero_grad()

            epoch_losses_list.append(loss.item())
            ep_vt.append(train_iter)
            iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                float().sum().item()
            iter_acc = float(iter_true_sample) / output.shape[0]
            # print("Train accuracy: {acc}".format(acc=iter_acc))
            # utils.writer.add_scalar("iter train acc", iter_acc, train_iter)
            # print(f"{train_iter} iter train acc: {iter_acc}")
            true_sample += iter_true_sample

        acc = (true_sample / train_size)*100
        # utils.writer.add_scalar("Epoch Acc", acc, e)
        # tqdm.write(f"Epoch:{e}  \t train_acc:{acc:.2f}")
        print("Train accuracy: {acc}".format(acc=acc))
#         plt.plot(ep_vt, epoch_losses_list, label='Train Loss')
#         plt.legend()
#         plt.show() 

        val_loss_list = []
        # ep_vt = []
        true_sample = 0
        model.eval()
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(val_loader):
                # val_iter += 1
                graph_feat, _ = model(features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                val_loss_list.append(loss.item())
                # ep_vt.append(val_iter)
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                true_sample += iter_true_sample
        acc = (true_sample / val_size)*100
        mx_acc[0] = max(mx_acc[0],acc)
#         print('Val Acc with Perturb: 0 :{}'.format(acc))

#         if args.save_model is True:
#           if (mx_acc >= 0.56) and (acc > mx_acc):
#             torch.save(model.state_dict(), PATH + '/' + args.dataSet + "/model_{}.torch".format(version))
#             version += 1

        test_loss_list = []
        true_sample = 0
        model.eval()
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(test_loader):
                test_iter += 1
                graph_feat, _ = model(features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                test_loss_list.append(loss.item())
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                # utils.writer.add_scalar("iter test acc", iter_acc, test_iter)
                # print(f"{test_iter} iter test acc: {iter_acc}")
                true_sample += iter_true_sample
        acc = (true_sample / test_size)*100
        # mx_acc = max(mx_acc,acc)
        en = time.time()
#         print("Required Time: ",en-st)
        # utils.writer.add_scalar("Epoch Acc", acc, e)
        # tqdm.write(f"Epoch:{e}  \t val_acc:{acc:.2f}")
#         print("Test accuracy with Perturb 0: {accuracy}".format(accuracy=acc))
        mx_acc[0] = max(mx_acc[0],acc)
    
#         mx_acc = 0
        model.eval()
        true_sample = 0
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(val_loader):
                # val_iter += 1
                c_features = feature_perturb(features, perturb_ratio=0.5)
                graph_feat, _ = model(c_features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                val_loss_list.append(loss.item())
                # ep_vt.append(val_iter)
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                true_sample += iter_true_sample
        acc = (true_sample / val_size)*100
        mx_acc[1] = max(mx_acc[1],acc)
#         print('Val Acc with  {}'.format(acc))

        # if args.save_model is True:
        #   if (mx_acc >= 0.65) and (acc > mx_acc):
        #     torch.save(model.state_dict(), PATH + '/' + args.dataSet + "/model_{}.torch".format(version))
        #     version += 1

        test_loss_list = []
        true_sample = 0
        model.eval()
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(test_loader):
                test_iter += 1
                c_features = feature_perturb(features, perturb_ratio=0.5)
                graph_feat, _ = model(features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                test_loss_list.append(loss.item())
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                # utils.writer.add_scalar("iter test acc", iter_acc, test_iter)
                # print(f"{test_iter} iter test acc: {iter_acc}")
                true_sample += iter_true_sample
        acc = (true_sample / test_size)*100
        mx_acc[1] = max(mx_acc[1],acc)
        en = time.time()
#         print("Required Time: ",en-st)
        # utils.writer.add_scalar("Epoch Acc", acc, e)
        # tqdm.write(f"Epoch:{e}  \t val_acc:{acc:.2f}")
#         print("Test accuracy: {accuracy}".format(accuracy=acc))


#         mx_acc = 0
        model.eval()
        true_sample = 0
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(val_loader):
                # val_iter += 1
                c_features = feature_perturb(features, perturb_ratio=1.0)
                graph_feat, _ = model(c_features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                val_loss_list.append(loss.item())
                # ep_vt.append(val_iter)
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                true_sample += iter_true_sample
        acc = (true_sample / val_size)*100
        mx_acc[2] = max(mx_acc[2],acc)
#         print('Val Acc {}'.format(acc))

        # if args.save_model is True:
        #   if (mx_acc >= 0.65) and (acc > mx_acc):
        #     torch.save(model.state_dict(), PATH + '/' + args.dataSet + "/model_{}.torch".format(version))
        #     version += 1

        test_loss_list = []
        true_sample = 0
        model.eval()
        true_sample = 0
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(test_loader):
                test_iter += 1
                c_features = feature_perturb(features, perturb_ratio=1.0)
                graph_feat, _ = model(features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                test_loss_list.append(loss.item())
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                # utils.writer.add_scalar("iter test acc", iter_acc, test_iter)
                # print(f"{test_iter} iter test acc: {iter_acc}")
                true_sample += iter_true_sample
        acc = (true_sample / test_size)*100
        mx_acc[2] = max(mx_acc[2],acc)
        en = time.time()
#         print("Required Time: ",en-st)
        # utils.writer.add_scalar("Epoch Acc", acc, e)
        # tqdm.write(f"Epoch:{e}  \t val_acc:{acc:.2f}")
#         print("Test accuracy: {accuracy}".format(accuracy=acc))

#         mx_acc = 0
        model.eval()
        true_sample = 0
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(val_loader):
                # val_iter += 1
                c_features = feature_perturb(features, perturb_ratio=1.5)
                graph_feat, _ = model(c_features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                val_loss_list.append(loss.item())
                # ep_vt.append(val_iter)
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                true_sample += iter_true_sample
        acc = (true_sample / val_size)*100
        mx_acc[3] = max(mx_acc[3],acc)
#         print('Val Acc {}'.format(acc))

        # if args.save_model is True:
        #   if (mx_acc >= 0.65) and (acc > mx_acc):
        #     torch.save(model.state_dict(), PATH + '/' + args.dataSet + "/model_{}.torch".format(version))
        #     version += 1

        test_loss_list = []
        true_sample = 0
        model.eval()
        true_sample = 0
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(test_loader):
                test_iter += 1
                c_features = feature_perturb(features, perturb_ratio=1.5)
                graph_feat, _ = model(features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                test_loss_list.append(loss.item())
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                # utils.writer.add_scalar("iter test acc", iter_acc, test_iter)
                # print(f"{test_iter} iter test acc: {iter_acc}")
                true_sample += iter_true_sample
        acc = (true_sample / test_size)*100
        mx_acc[3] = max(mx_acc[3],acc)
        en = time.time()
#         print("Required Time: ",en-st)
        # utils.writer.add_scalar("Epoch Acc", acc, e)
        # tqdm.write(f"Epoch:{e}  \t val_acc:{acc:.2f}")
#         print("Test accuracy: {accuracy}".format(accuracy=acc))

#         mx_acc = 0
        model.eval()
        true_sample = 0
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(val_loader):
                # val_iter += 1
                c_features = feature_perturb(features, perturb_ratio=2.0)
                graph_feat, _ = model(c_features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                val_loss_list.append(loss.item())
                # ep_vt.append(val_iter)
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                true_sample += iter_true_sample
        acc = (true_sample / val_size)*100
        mx_acc[4] = max(mx_acc[4],acc)
#         print('Val Acc {}'.format(acc))

        # if args.save_model is True:
        #   if (mx_acc >= 0.65) and (acc > mx_acc):
        #     torch.save(model.state_dict(), PATH + '/' + args.dataSet + "/model_{}.torch".format(version))
        #     version += 1

        test_loss_list = []
        true_sample = 0
        model.eval()
        with torch.no_grad():
            for i, (adj, features,batch_labels) in enumerate(test_loader):
                test_iter += 1
                c_features = feature_perturb(features, perturb_ratio=2.0)
                graph_feat, _ = model(features, adj)
                output = model.classifier(graph_feat)
                loss = model.loss(output, batch_labels)
                test_loss_list.append(loss.item())
                iter_true_sample = (output.argmax(dim=1).long() == batch_labels.long()). \
                    float().sum().item()
                iter_acc = float(iter_true_sample) / output.shape[0]
                # utils.writer.add_scalar("iter test acc", iter_acc, test_iter)
                # print(f"{test_iter} iter test acc: {iter_acc}")
                true_sample += iter_true_sample
        acc = (true_sample / test_size)*100
        mx_acc[4] = max(mx_acc[4],acc)
        en = time.time()
#         print("Required Time: ",en-st)
        # utils.writer.add_scalar("Epoch Acc", acc, e)
        # tqdm.write(f"Epoch:{e}  \t val_acc:{acc:.2f}")
#         print("Test accuracy: {accuracy}".format(accuracy=acc))
        results = []
        results.append(round(mx_acc[0], 4))
        results.append(round(mx_acc[1], 4))
        results.append(round(mx_acc[2], 4))
        results.append(round(mx_acc[3], 4))
        results.append(round(mx_acc[4], 4))
        
        print("Best accuracy without perturbation: ",results[0])
        print("Best accuracy with 0.5 perturbation: ",results[1])
        print("Best accuracy with 1.0 perturbation: ",results[2])
        print("Best accuracy with 1.5 perturbation: ",results[3])
        print("Best accuracy with 2.0 perturbation: ",results[4])
    result_file = ffopen(file_name, 'a',title)
    buffer = [str(d) for d in results]
    buf = ','.join(buffer) + '\n'
    result_file.write(buf)
    result_file.close()<jupyter_output><empty_output><jupyter_text>File Management<jupyter_code>import time,os,errno
            

def ffopen(fileLocation, mode, title=None):
  # if not os.path.exists(os.path.dirname(fileLocation)):
  if not os.path.exists(fileLocation):
    try:
      os.makedirs(os.path.dirname(fileLocation))
    except OSError as exc:  # Guard against race condition
      if exc.errno != errno.EEXIST:
        raise
    f = open(fileLocation, mode)
    if title is not None:
      f.write(title)
    f.close()

  f = open(fileLocation, mode)
  return f<jupyter_output><empty_output><jupyter_text>Execution<jupyter_code>if __name__ == '__main__':
  seed = 42
  
  random.seed(seed)
	
  
  np.random.seed(seed)
	
  
  torch.manual_seed(seed)
	
  
  torch.cuda.manual_seed_all(seed)
  
  
  main()


from google.colab import drive
drive.mount('/content/drive')<jupyter_output><empty_output><jupyter_text>Feature Attacks<jupyter_code>import torch

m = Normal(torch.tensor([[0.0, 1.0], [1.0, 0.0]]), torch.tensor([[1.0, 0.5],[0.5, 1.0]]))
print(m.sample())
a = torch.rand(4, 5)
print(a)
b = torch.rand(3, 5)
print(b)
c = torch.cat([a, b], dim=0)
print(c)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1312811349361057, 'fraction_numerical': 0.01735434264674031, 'mean_word_length': 2.3460458374852795, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 58, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '352625', 'license': 'MIT', 'n_tokens_mistral': 12921, 'n_tokens_neox': 12417, 'n_words': 2137, 'path': 'HIBPool.ipynb', 'repo_name': 'forkkr/HIBPool', 'size': '36936'}