<jupyter_start><jupyter_text>Prepare additional dataset for inverse problem-solving- Four types of "inverse problem" graphs will be made<jupyter_code>import sys
sys.path.append("../MIGraph/GraphConv/")

from ValueTransformer import ValueTransformer
from ConvGraphScript import drawGraph,checkGraphList
from AutoParameterScaling import AutoParameterScaling
from ConvGraphmlToGraph import loadGraphCSV
from PrepGraphScript import PrepGraphScript

import glob
import os
import joblib

os.chdir("praparingGraphs")
AutoSC=joblib.load("output/AutoSC.scaler")

folderList=glob.glob("screen_input/*")
print(folderList)
#get folder list
CSVPathList=[]
graphPathList=[]

for folder in folderList:
    CSVPath=folder+"/"+os.path.basename(folder)+".csv"
    graphPath=folder+"/graph/"
    CSVPathList.append(CSVPath)
    graphPathList.append(graphPath)
    
#prepare graphgs
VT=ValueTransformer()
for CSVPath,graphPath in zip(CSVPathList,graphPathList):
    print(CSVPath)
    gList=loadGraphCSV(CSVPath,graphPath)
    gList=VT.convertGraphList(gList)
    checkGraphList(gList)
    
    #save
    filename=os.path.basename(CSVPath)
    outname="screen_output/"+filename.replace(".csv","")+".graphbin"
    print("saving...", outname)
    joblib.dump(gList,outname,compress=3)
    

#scaling
folderList=glob.glob("screen_output/*.graphbin")

#for i in folderList:
#    AutoSC.autoTransform([i])    
AutoSC.autoTransform(folderList)   
from tqdm import tqdm

#check graph
graphbinList=glob.glob("screen_output/*.graphbin")

gList=[]
for file in tqdm(graphbinList):
    print(file)
    temp=joblib.load(file)
    gList.extend(temp)

print(len(gList), " plots")

drawGraph(gList[-2])
g=gList[-2]
nodeVals=[g.nodes[node]["label"] for node in g.nodes]
nodeVals<jupyter_output>100%|██████████| 1/1 [00:00<00:00, 230.58it/s]
{'fraction_non_alphanumeric': 0.13033832501386577, 'fraction_numerical': 0.011647254575707155, 'mean_word_length': 5.7819548872180455, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '648090', 'license': 'MIT', 'n_tokens_mistral': 657, 'n_tokens_neox': 631, 'n_words': 113, 'path': 'codes/DEMO3_Inverse_problem/step1_b_PrepareSpecialDataset.ipynb', 'repo_name': 'KentaroKutsukake/Integrating-multiple-materials-science-projects', 'size': '1803'}