<jupyter_start><jupyter_text>Notebook for preparing and saving MOLECULAR (ZINC full) graphs<jupyter_code>import numpy as np
import torch
import pickle
import time
import os
%matplotlib inline
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Download ZINC full dataset<jupyter_code>if not os.path.isfile('molecules_zinc_full.zip'):
    print('downloading..')
    !curl https://www.dropbox.com/s/grhitgnuuixoxwl/molecules_zinc_full.zip?dl=1 -o molecules_zinc_full.zip -J -L -k
    !unzip molecules_zinc_full.zip -d ../
    !rm -r ../__MACOSX/
    # !tar -xvf molecules_zinc_full.zip -C ../
else:
    print('File already downloaded')<jupyter_output>downloading..
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100 43.6M  100 43.6M    0     0  7898k      0  0:00:05  0:00:05 --:--:-- 10.3M
Archive:  molecules_zinc_full.zip
   creating: ../molecules/zinc_full/
  inflating: ../__MACOSX/molecules/._zinc_full  
  inflating: ../molecules/zinc_full/test.pickle  
  inflating: ../__MACOSX/molecules/zinc_full/._test.pickle  
  inflating: ../molecules/zinc_full/.DS_Store  
  inflating: ../__MACOSX/molecules/zinc_full/._.DS_Store  
  inflating: ../molecules/zinc_full/val.pickle  
  inflating: ../__MACOSX/molecules/zinc_full/._val.pickle  
  inflating: ../molecules/zinc_full/atom_dict.pickle  
  inflating: ../__MACOSX/molecules/zinc_full/._atom_dict.pickle  
  inflating: ../molecul[...]<jupyter_text>Convert to DGL format and save with pickle<jupyter_code>import os
os.chdir('../../') # go to root folder of the project
print(os.getcwd())

import pickle

%load_ext autoreload
%autoreload 2

from data.molecules import MoleculeDatasetDGL 

from data.data import LoadData
from torch.utils.data import DataLoader
from data.molecules import MoleculeDataset

DATASET_NAME = 'ZINC-full'
dataset = MoleculeDatasetDGL(DATASET_NAME)
def plot_histo_graphs(dataset, title):
    # histogram of graph sizes
    graph_sizes = []
    for graph in dataset:
        graph_sizes.append(graph[0].number_of_nodes())
    plt.figure(1)
    plt.hist(graph_sizes, bins=20)
    plt.title(title)
    plt.show()
    graph_sizes = torch.Tensor(graph_sizes)
    print('min/max :',graph_sizes.min().long().item(),graph_sizes.max().long().item())
    
plot_histo_graphs(dataset.train,'trainset')
plot_histo_graphs(dataset.val,'valset')
plot_histo_graphs(dataset.test,'testset')

print(len(dataset.train))
print(len(dataset.val))
print(len(dataset.test))

print(dataset.train[0])
print(dataset.val[0])
print(dataset.test[0])

num_atom_type = 28
num_bond_type = 4

start = time.time()
with open('data/molecules/ZINC-full.pkl','wb') as f:
        pickle.dump([dataset.train,dataset.val,dataset.test,num_atom_type,num_bond_type],f)
print('Time (sec):',time.time() - start) #131s<jupyter_output>/home/vijay/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/torch/storage.py:34: FutureWarning: pickle support for Storage will be removed in 1.5. Use `torch.save` instead
  warnings.warn("pickle support for Storage will be removed in 1.5. Use `torch.save` instead", FutureWarning)<jupyter_text>Test load function<jupyter_code>DATASET_NAME = 'ZINC-full'
dataset = LoadData(DATASET_NAME)
trainset, valset, testset = dataset.train, dataset.val, dataset.test #154s
batch_size = 10
collate = MoleculeDataset.collate
print(MoleculeDataset)
train_loader = DataLoader(trainset, batch_size=batch_size, shuffle=True, collate_fn=collate)<jupyter_output><class 'data.molecules.MoleculeDataset'>
{'fraction_non_alphanumeric': 0.13251402618220678, 'fraction_numerical': 0.021106064654020838, 'mean_word_length': 3.697616060225847, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '648364', 'license': 'MIT', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1339, 'n_words': 259, 'path': 'data/molecules/prepare_molecules_ZINC_full.ipynb', 'repo_name': 'mzl9039/benchmarking-gnns', 'size': '3743'}