<jupyter_start><jupyter_text>load data<jupyter_code>from tqdm.notebook import tqdm, trange
import networkx as nx
import pickle
import numpy as np
import tensorflow as tf
import torch
print(tf.__version__)

train_freq_mat = pickle.load(open(f'graph_data/GKAT_freq_mats_train_len={walk_len}.pkl', 'rb'))
val_freq_mat = pickle.load(open(f'graph_data/GKAT_freq_mats_val_len={walk_len}.pkl', 'rb'))
# load train and validation graph data, with 1535 samples for training, and 512 samples for validation
# each graph contains 50 nodes

num_nodes = 50
num_train = 1535
num_val = 512


# load all train and validation graphs
train_graphs = pickle.load(open(f'graph_data/train_graphs.pkl', 'rb'))
val_graphs = pickle.load(open(f'graph_data/val_graphs.pkl', 'rb'))

# load all labels
train_labels = np.load('graph_data/train_labels.npy')
val_labels = np.load('graph_data/val_labels.npy')

# save random walk lists as pickle file
a_file = open("graph_data/train_graphs.pkl", "wb")
pickle.dump(train_graphs, a_file)
a_file.close()

a_file = open("graph_data/val_graphs.pkl", "wb")
pickle.dump(val_graphs, a_file)
a_file.close()<jupyter_output><empty_output><jupyter_text>generate random walks<jupyter_code># generate random walks for GKAT

from deepwalk import OnlyWalk

path_length = 10
num_random_walk= 50



def generate_walks_GKAT(graphs, num_random_walk, path_length, stopping_prob = 0.0, p=1, q=1, ignore_start = False):

  walks = []
  
  print('Start generating GWK masking')
  print("walk length = ", path_length)
  print("number of random walks = ", num_random_walk)

  for i in tqdm(range(len(graphs))):
      
      graph = (graphs[i])
      n2v = OnlyWalk.Node2vec_onlywalk(graph = graph, path_length=path_length, num_paths=num_random_walk, p=p, q=q, stop_prob = stopping_prob, with_freq_mat = True)
      walks.append(n2v.walker.walks_dict)

  return walks

# start random walks
GKAT_walks_train = generate_walks_GKAT(train_graphs, path_length = path_length, num_random_walk = num_random_walk, stopping_prob = stopping_prob, p = p, q= q, ignore_start = ignore_start)
GKAT_walks_val = generate_walks_GKAT(val_graphs, path_length = path_length, num_random_walk = num_random_walk, stopping_prob = stopping_prob, p = p, q= q, ignore_start = ignore_start)

# save random walk lists as pickle file
a_file = open("graph_data/GKAT_walks_dict_train.pkl", "wb")
pickle.dump(GKAT_walks_train, a_file)
a_file.close()

a_file = open("graph_data/GKAT_walks_dict_val.pkl", "wb")
pickle.dump(GKAT_walks_val, a_file)
a_file.close()<jupyter_output><empty_output><jupyter_text>generate random walk frequency matrix and GKAT dot product kernel<jupyter_code>def generate_trunc_walks_dict(walks, trunc_len):
  print("generating walks with length = ", trunc_len)
  num_random_walk = len(walks[0][0])
  num_nodes = len(walks[0])

  trunc_walks = []

  num_graphs = len(walks)
  for i in range(num_graphs):
    g_dict = {}
    for j in range(num_nodes):
      walklist = []
      for k in range(num_random_walk):
        walklist.append(walks[i][j][k][:trunc_len])
      g_dict[j] = walklist
    trunc_walks.append(g_dict)
  return trunc_walks


def generate_frequency_matrix_and_masking_GKAT(walks_dict):

  num_graphs = len(walks_dict)
  num_random_walk = len(walks_dict[0][0])
  num_nodes = len(walks_dict[0])
  walk_length = len(walks_dict[0][0][0])

  freq_mat_list = []
  dot_kernel_list = []

  for graph in tqdm(walks_dict):

    freq_mat = np.zeros([num_nodes, num_nodes])

    for key in graph:
      for i in range(num_random_walk):
        for j in range(walk_length):
          freq_mat[int(key),int(graph[key][i][j])] +=1
    freq_mat /= num_random_walk

    dot_prod = np.matmul(freq_mat, np.transpose(freq_mat))

    # divide the dot_prod kernel by the norm of the kernel
    deno = np.matmul(np.diagonal(dot_prod)[:, None], np.transpose(np.diagonal(dot_prod)[:, None]))
    dot_kernel = dot_prod / np.sqrt(deno)  #np.diagonal(dot_prod)[:, None]

    freq_mat_list.append(freq_mat* num_random_walk)
    dot_kernel_list.append(dot_kernel)

  return freq_mat_list, dot_kernel_list



# generate random walk frequency matrix and GKAT dot product kernel with different random walk lengths
# the generated data are saved also as pickle files

for trunc_len in range(2, 11):

  trunc_walks_train = generate_trunc_walks_dict(GKAT_walks_train, trunc_len)
  trunc_walks_val = generate_trunc_walks_dict(GKAT_walks_val, trunc_len)



  freq_mat_list, dot_kernel_list = generate_frequency_matrix_and_masking_GKAT(trunc_walks_train)

  a_file = open(f"graph_data/GKAT_freq_mats_train_len={trunc_len}.pkl", "wb")
  pickle.dump(freq_mat_list, a_file)
  a_file.close()

  a_file = open(f"graph_data/GKAT_dot_kernels_train_len={trunc_len}.pkl", "wb")
  pickle.dump(dot_kernel_list, a_file)
  a_file.close()




  freq_mat_list, dot_kernel_list = generate_frequency_matrix_and_masking_GKAT(trunc_walks_val)

  a_file = open(f"graph_data/GKAT_freq_mats_val_len={trunc_len}.pkl", "wb")
  pickle.dump(freq_mat_list, a_file)
  a_file.close()

  a_file = open(f"graph_data/GKAT_dot_kernels_val_len={trunc_len}.pkl", "wb")
  pickle.dump(dot_kernel_list, a_file)
  a_file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10954404945904173, 'fraction_numerical': 0.00811437403400309, 'mean_word_length': 5.662805662805662, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '501113', 'license': 'MIT', 'n_tokens_mistral': 2050, 'n_tokens_neox': 1925, 'n_words': 374, 'path': 'Synthetic_InducedCycle/Generate_GKAT_masking_and_walks.ipynb', 'repo_name': 'GKAT-NeurIPS2021/GKAT_Experiments', 'size': '5176'}