<jupyter_start><jupyter_text>This script convert graph dataset to adjacency matix and node vectors for machine learning<jupyter_code>import sys
sys.path.append("../MIGraph/GraphConv/")
sys.path.append("../MIGraph/Encoders/")
import numpy as np
import pandas as pd
from tqdm import tqdm
import joblib
import warnings
from Config import Config
import glob
from ConvGraphScript import drawGraph
from PrepDataset import PrepDataset

%load_ext autoreload
%autoreload 2
    
warnings.filterwarnings('ignore')
CF=Config()

print("mode: ",CF.genre)
print("targetParamMode: ",CF.targetParamMode)
#init compound encoder
PD=PrepDataset()
allCompundsPath="praparingGraphs/output/allcompounds.csv.gz"

PD.setCompoundEncoder(allCompundsPath)
#load graph files
graphbinList=glob.glob("praparingGraphs/output/*.graphbin")
print(graphbinList)
graphbinList.sort(reverse=True)

for file in tqdm(graphbinList):
    print(file)
    gl=joblib.load(file)
    PD.graphList.extend(gl)
PD.setScaler()
print(len(PD.graphList))<jupyter_output>load BERT encoder
1581<jupyter_text>preparing problems- in demo 1, only conducitvity was set as problems.- on the other hand, other parametes, such as melting temperature, will be set as problems.- see "self.targetParams" in Config.py to check what kind of parameters can be problems.<jupyter_code>#prepare dataset

#parallel mode off
PD.parallel=False
trainDataset=PD.convToDataset(CF.genre,graphList=PD.graphList)


joblib.dump(trainDataset,"bin/traindataset.bin",compress=9)

#see how converted...
trainDataset[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09993593850096093, 'fraction_numerical': 0.005124919923126201, 'mean_word_length': 6.619512195121951, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '538651', 'license': 'MIT', 'n_tokens_mistral': 521, 'n_tokens_neox': 509, 'n_words': 130, 'path': 'codes/DEMO2_Multitask_learning/step2_ConvertToMatrx.ipynb', 'repo_name': 'KentaroKutsukake/Integrating-multiple-materials-science-projects', 'size': '1561'}