<jupyter_start><jupyter_text>Imports<jupyter_code>from dmsbatch import create_batch_client, create_blob_client<jupyter_output><empty_output><jupyter_text>First create a batch client from the config file<jupyter_code>client = create_batch_client('../tests/data/dmsbatch.config')
blob_client = create_blob_client('../tests/data/dmsbatch.config')<jupyter_output><empty_output><jupyter_text>Application packagesTo copy large files and programs it is best to zip (or targz) them and upload them as application packagesApplication packages are setup separately in either azure management apis or from the web console or cli toolThese are referenced here by their name and versione.g. DSM2, python and other programs<jupyter_code>app_pkgs = [('dsm2linux', '8.2.8449db2', 'DSM2-8.2.8449db2-Linux/bin')]<jupyter_output><empty_output><jupyter_text>Create or resize existing poolIf the pool doesn't exist it will create itIf the pool exists, it will resize to the second arg<jupyter_code>pool_start_cmds = ['printenv',
'yum install -y glibc.i686 libstdc++.i686 glibc.x86_64 libstdc++.x86_64',# --setopt=protected_multilib=false',
'yum-config-manager --add-repo https://yum.repos.intel.com/2019/setup/intel-psxe-runtime-2019.repo',
'rpm --import https://yum.repos.intel.com/2019/setup/RPM-GPG-KEY-intel-psxe-runtime-2019',
'yum install -y intel-icc-runtime-32bit intel-ifort-runtime-32bit']
client.wrap_commands_in_shell('linux',pool_start_cmds)
created = client.create_pool('dsm2linuxpool',
                             1,
                             app_packages=[(app, version)
                                           for app, version, _ in app_pkgs],
                             vm_size='standard_d2d_v4',  # vm_size='standard_f2s_v2',
                             tasks_per_vm=2,
                             os_image_data=('openlogic', 'centos', '7_8'),
                             start_task_cmd=client.wrap_commands_in_shell(
                                 pool_start_cmds, ostype='linux'),
                             start_task_admin=True,
                             elevation_level='admin'
                             )
if not created:
    client.resize_pool('dsm2linuxpool',1)
client.wait_for_pool_nodes('dsm2linuxpool')<jupyter_output><empty_output><jupyter_text>Create job on pool or fail if it existsJobs are containers of tasks (things that run on nodes (machines) in the pool). If this exists, the next line will fail<jupyter_code>try:
    client.create_job('dsm2linuxjobs','dsm2linuxpool')
except Exception as exc:
    print('Job already exists? Will continue anyways')<jupyter_output><empty_output><jupyter_text>Upload input filesZip the input files and add them to storage container (in this case the auto storage associated with the batch account)<jupyter_code>import os, datetime
userid = os.getlogin()
tsnow = str(datetime.datetime.now().timestamp()).split('.')[0]
task_name = f'hydro_run_{tsnow}_{userid}'
print(task_name)
local_dir = '../tests/data/dsm2v821'
input_file=blob_client.zip_and_upload('dsm2linuxjobs',f'{task_name}/dsm2v821.zip',local_dir,30)
input_file = client.create_input_file_spec('dsm2linuxjobs',blob_prefix=f'{task_name}/dsm2v821.zip',file_path='.')
import dmsbatch
permissions = dmsbatch.commands.BlobSasPermissions(write=True)
# |helpers.azureblob.BlobPermissions.ADD|helpers.azureblob.BlobPermissions.CREATE
output_dir_sas_url = blob_client.get_container_sas_url('dsm2linuxjobs', permissions)
print(output_dir_sas_url)
std_out_files = client.create_output_file_spec('../std*.txt', output_dir_sas_url, blob_path=f'{task_name}')
output_dir = client.create_output_file_spec('**/output/*',output_dir_sas_url, blob_path=f'{task_name}')<jupyter_output><empty_output><jupyter_text>Create a taskThis uses the application package as pre -set up. If not, create one https://docs.microsoft.com/en-us/azure/batch/batch-application-packages<jupyter_code># tsnow = str(datetime.datetime.now().timestamp()).split('.')[0]
# task_name = f'hydro_run_{tsnow}'
set_path_string = client.set_path_to_apps(app_pkgs, ostype='linux')
cmd_string = client.wrap_cmd_with_app_path(f"""source /opt/intel/psxe_runtime/linux/bin/compilervars.sh ia32;
{set_path_string};
cd {task_name};
unzip dsm2v821.zip;
rm dsm2v821.zip;
cd study_templates/historical;
hydro hydro.inp;
qual qual_ec.inp;
    """, app_pkgs, ostype='linux')
hydro_task=client.create_task(task_name, cmd_string, 
    resource_files=[input_file], output_files=[std_out_files, output_dir])

task_name<jupyter_output><empty_output><jupyter_text>Next submit the task and wait<jupyter_code>client.submit_tasks('dsm2linuxjobs',[hydro_task])
client.wait_for_tasks_to_complete('dsm2linuxjobs',timeout=datetime.timedelta(minutes=80),polling_interval_secs=60)<jupyter_output><empty_output><jupyter_text>Finally resize the pool to 0 to save costs<jupyter_code>client.resize_pool('dsm2linuxpool',0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11356209150326797, 'fraction_numerical': 0.021037581699346407, 'mean_word_length': 4.957420924574209, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '260267', 'license': 'MIT', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1643, 'n_words': 291, 'path': 'notebooks/sample_submit_dsm2_linux_historical.ipynb', 'repo_name': 'dwr-psandhu/azure_dms_batch', 'size': '4896'}