<jupyter_start><jupyter_text>Launch pytorch traing jobsCode to create a batch script for launching jobs on cori GPUSep 1, 2020<jupyter_code>import os
import glob,time
import subprocess as sp
import numpy as np

curr_dir=os.getcwd()
print(curr_dir)
code_dirs={'128':'/global/u1/v/vpa/project/jpt_notebooks/Cosmology/Cosmo_GAN/repositories/cosmogan_pytorch/code/1_basic_GAN/1_main_code',
          '512':'/global/u1/v/vpa/project/jpt_notebooks/Cosmology/Cosmo_GAN/repositories/cosmogan_pytorch/cosmogan/1_main_code' }

config={'128': '/global/u1/v/vpa/project/jpt_notebooks/Cosmology/Cosmo_GAN/repositories/cosmogan_pytorch/code/1_basic_GAN/1_main_code/config_128.yaml',
        '512': '/global/u1/v/vpa/project/jpt_notebooks/Cosmology/Cosmo_GAN/repositories/cosmogan_pytorch/code/1_basic_GAN/1_main_code/config_256.yaml'}
### Default dictionary
img_size=128
dict_pars={'epochs':20, 'batch_size':64, 'nodes':1,'seed':36723705, 'spec_loss': '','ip_fldr':'-',
           'job_name':'lambda2.0',
#            'ip_fldr':'/global/cfs/cdirs/m3363/vayyar/cosmogan_data/results_from_other_code/pytorch/results/128sq/20201014_073327/',
           'spec_loss':'--specloss',
           'deterministic':'--deterministic',
           'save_steps_list':'22010 33550 35970 50810 60320 79050',
           'lr':0.0002,
           'lambda1':2.0,
           'procs':8, 'time':'4:00:00','mode':'fresh'}

dict_pars['code_dir']=code_dirs[str(img_size)]
dict_pars['config_file']=config[str(img_size)]
print(dict_pars)
bash_strg='''#!/bin/bash
#################
#SBATCH --nodes={nodes}
#SBATCH --qos=regular
#SBATCH --output=slurm-%x-%j.out
#SBATCH --constraint=gpu
#SBATCH --account=m3363
#SBATCH --ntasks-per-node={procs}
#SBATCH --cpus-per-task=8 
#SBATCH --gpus-per-task=1
#SBATCH --time={time}
#SBATCH --job-name={job_name}

echo "--start date" `date` `date +%s`
echo '--hostname ' $HOSTNAME
### Initial setup

conda activate v3
### module load pytorch-v1.4.0-gpu
##module load esslurm

code_dir={code_dir}
config_file={config_file}
python $code_dir/main.py --seed {seed} {spec_loss} {deterministic} -l {save_steps_list} --config $config_file --mode {mode} --ngpu {procs} --learn_rate {lr} --batchsize {batch_size} --epochs {epochs} -ld1 {lambda1} -ip {ip_fldr} -rs {job_name}

echo "--end date" `date` `date +%s`
'''.format(**dict_pars)
bash_strg
staging_loc='/global/cfs/cdirs/m3363/vayyar/cosmogan_data/results_from_other_code/pytorch/results/'
fname='batch_train.sh'
filename=staging_loc+fname
with open (filename,'w') as f:
    f.write(bash_strg)
print(filename)
### Move to staging locations in project space:
os.chdir(staging_loc)<jupyter_output><empty_output><jupyter_text>Submit job to cori GPU<jupyter_code>%%bash -s "$filename" ## Use python variable in bash
cat $1
chmod +x $1
%%bash -s "$filename" ## Use python variable in bash
module load cgpu
sbatch $1<jupyter_output>Submitted batch job 1471256
{'fraction_non_alphanumeric': 0.16505190311418685, 'fraction_numerical': 0.04948096885813149, 'mean_word_length': 5.151063829787234, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '627237', 'license': 'BSD-3-Clause-LBNL', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1147, 'n_words': 188, 'path': 'code/1_basic_GAN/launch_train_pytorch.ipynb', 'repo_name': 'vmos1/cosmogan_pytorch', 'size': '2890'}