<jupyter_start><jupyter_text>Launch LBANN traing jobsCode to create a batch script for launching jobs on cori GPUSep 1, 2020<jupyter_code>import os
import glob,time
import subprocess as sp
import numpy as np

import yaml

from ipywidgets import *
start_dir=os.getcwd()
print(start_dir)<jupyter_output>/global/u1/v/vpa/project/jpt_notebooks/Cosmology/Cosmo_GAN/repositories/cosmogan_pytorch/code/run_scripts<jupyter_text>Define machine and code<jupyter_code># facility='cori'
# img_size=64
# run_type='3dgan'
# assert facility in ['cori','summit'], "Error%s"%(facility)
# assert img_size in [64,128,512]
# assert run_type in ['2dgan','2dcgan','3dgan','3dcgan']
def f(facility,img_size,run_type): return facility,img_size,run_type
w=interactive(f,
                facility=ToggleButtons(options=['cori','summit']),
                img_size=ToggleButtons(options=[64,128,512]),
                run_type=ToggleButtons(options=['2dgan','2dcgan','3dgan','3dcgan']) )
display(w)

result=w.result
facility,img_size,run_type=result
print(facility,img_size,run_type)<jupyter_output>cori 64 3dcgan<jupyter_text>Read ref_launch.yaml and define dictionary<jupyter_code># start_dir='/global/u1/v/vpa/project/jpt_notebooks/Cosmology/Cosmo_GAN/repositories/cosmogan_pytorch/code/run_scripts'
launch_cfile=start_dir+'/ref_launch.yaml'

with open(launch_cfile) as f:
    config_dict= yaml.load(f, Loader=yaml.SafeLoader)
## Read ref_launch.yaml
main_code_dir=config_dict[facility]['main_code_dir']
dict_pars=config_dict[facility][img_size][run_type]
dict_pars.update({'nodes':1,'job_name':'analysis_'+run_type,
                 'queue':'debug','bins':'unenven','cores':64,'time':'00:30:00'})

dict_pars['code_dir']=main_code_dir+dict_pars['code_dir']
dict_pars['staging_loc']=config_dict[facility]['staging_loc']
dict_pars['img_size']=img_size
print(dict_pars)

staging_loc=dict_pars['staging_loc']
op_fldr=dict_pars['staging_loc']+dict_pars['op_dir']
op_fldr
# %%bash -s "$op_fldr"
# ls $1
# parent_dir=u.result
dir_lst=[i.split('/')[-1] for i in glob.glob(op_fldr+'20210*')]
q=interactive(lambda x: x, x=ToggleButtons(options=dir_lst))
display(q)
# ip_folder='20210507_121514_cgan_bs8_lr0.00005_nodes8'
ip_folder=q.result
dict_pars['ip_folder']=dict_pars['staging_loc']+dict_pars['op_dir']+ip_folder
print(dict_pars['ip_folder'])
bash_strg='''#!/bin/bash
#################
#SBATCH --nodes=1
#SBATCH --qos={queue}
#SBATCH --output=slurm-%x-%j.out
#SBATCH --account=m3363
#SBATCH -C haswell
#SBATCH --time={time}
#SBATCH --job-name={job_name}

echo "--start date" `date` `date +%s`
echo '--hostname ' $HOSTNAME

conda activate v3
code_dir={code_dir}

python $code_dir/post_analysis_pandas.py -f {ip_folder}  -v {val_file} -c {cores} -bin {bins} -i {img_size}
conda deactivate
echo "--end date" `date` `date +%s`
'''.format(**dict_pars)
fname='batch_analysis_{0}.sh'.format(run_type)
filename=dict_pars['staging_loc']+fname
with open (filename,'w') as f:
    f.write(bash_strg)
print(filename)
## Move to staging locations in project space:
os.chdir(dict_pars['staging_loc'])
%%bash -s "$filename" ## Use python variable in bash
cat $1
chmod +x $1<jupyter_output>#!/bin/bash
#################
#SBATCH --nodes=1
#SBATCH --qos=debug
#SBATCH --output=slurm-%x-%j.out
#SBATCH --account=m3363
#SBATCH -C haswell
#SBATCH --time=00:30:00
#SBATCH --job-name=analysis_3dcgan

echo "--start date" `date` `date +%s`
echo '--hostname ' $HOSTNAME

conda activate v3
code_dir=/global/u1/v/vpa/project/jpt_notebooks/Cosmology/Cosmo_GAN/repositories/cosmogan_pytorch/code/5_3d_cgan/1_main_code

python $code_dir/post_analysis_pandas.py -f /global/cfs/cdirs/m3363/vayyar/cosmogan_data/results_from_other_code/pytorch/results/3d_cGAN/20210726_173009_cgan_128_nodes1_lr0.000002_finetune  -v /global/cfs/cdirs/m3363/vayyar/cosmogan_data/raw_data/3d_data/dataset4_smoothing_4univ_cgan_varying_sigma_128cube/ -c 64 -bin unenven -i 128
conda deactivate
echo "--end date" `date` `date +%s`<jupyter_text>Submit job to cori GPU<jupyter_code>%%bash -s "$filename" ## Use python variable in bash
# module load cgpu
sbatch $1
pwd<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15884213086840185, 'fraction_numerical': 0.03575772318170761, 'mean_word_length': 7.015594541910332, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '608232', 'license': 'BSD-3-Clause-LBNL', 'n_tokens_mistral': 1757, 'n_tokens_neox': 1648, 'n_words': 259, 'path': 'code/run_scripts/launch_compute_pytorch.ipynb', 'repo_name': 'vmos1/cosmogan_pytorch', 'size': '4111'}