<jupyter_start><jupyter_text>Create .yaml file of dictionary with filenames sorted by oil type and whether completed or in queue<jupyter_code>import numpy as np
import pandas as pd
import os
import pathlib
import xarray as xr
import yaml
from matplotlib import pyplot as plt, cm, colors
from cmocean import cm as cmo
from datetime import datetime
from glob import glob
#from tqdm.notebook import tqdm

%matplotlib inline
plt.rcParams.update({'font.size': 12, 'axes.titlesize': 12})
# Results path
results_dir = '/scratch/dlatorne/MIDOSS/runs/monte-carlo'
from glob import glob
runsets = sorted(glob(os.path.join(results_dir,"near-BP_*")))
runsets
runsets_5 = runsets[:5]
runsets_5
#sorted(glob(os.path.join('/scratch/dlatorne/MIDOSS/runs/monte-carlo/near-BP_10th-100_2021-09-25T162640/results/*')))
runset = runsets_5[0]
list.append(list)
list[0]
runs = []
for runset in runsets_5:
    runs.extend(sorted(glob(os.path.join(runset,'results','near-BP_*')))[:])
netcdf_files = []
for run in runs:
    netcdf_files.extend(sorted(glob(os.path.join(run,'Lagrangian*.nc')))[:])
netcdf_files[0]
oiltype = netcdf_files[0].split('/')[-1].split('_')[1].split('-')[0]
oiltype
oil_types = [
    'akns', 
    'bunker', 
    'dilbit', 
    'jet', 
    'diesel', 
    'gas', 
    'other'
]
file_boolean = {}
files = {}
file_boolean[oil_types[0]] = [oil_types[0] in file for file in netcdf_files]
files[oil_types[0]]=[file for i,file in enumerate(netcdf_files)  if file_boolean[oil_types[0]][i]]
files[oil_types[0]]<jupyter_output><empty_output><jupyter_text>time to pull all this together<jupyter_code>def get_MOHID_netcdf_filenames(results_dir, output_dir):
    """Get lists of filepaths and filenames for netcdf files of model output, 
    grouped by oil types. NOTE: jet and gas are run as diesel; other is run 
    as bunker.  
    
    :param str results_dir: File path for root directory of run sets. 
    On Graham, the filepath is `/scratch/dlatorne/MIDOSS/runs/monte-carlo`
    
    :param str output_dir: File path for storing MOHID_results_locations_{date}.yaml,
    which contains file paths for completed runs, sorted by oil type.  
    
    :return: Dataframe of file paths and names, sorted by oil types, namely: 
    akns, bunker, dilbit, jet, diesel, gas and other.  Note: jet and gas are 
    run as diesel; other is run as bunker.  
    :rtype: :py:class:`pandas.DataFrame`
    """
    oil_types = [
        'akns', 
        'bunker', 
        'dilbit', 
        'jet', 
        'diesel', 
        'gas', 
        'other'
    ]
    # get list of runsets
    runsets = sorted(glob(os.path.join(results_dir,"near-BP_*")))
    # get list of runs within each runset
    runs = []
    for runset in runsets:
        runs.extend(sorted(
            glob(os.path.join(runset,'results','near-BP_*')))[:])        
    # get complete list of netcdf files
    netcdf_files = []
    for run in runs:
        netcdf_files.extend(sorted(
            glob(os.path.join(run,'Lagrangian*.nc')))[:])
    # sort filenames by oil type.  
    file_boolean = {}
    files = {}
    files['all'] = []
    for oil in oil_types:
        file_boolean[oil] = [oil in file for file in netcdf_files]
        files[oil]='test'#[file for i,file in enumerate(netcdf_files) \
                    #if file_boolean[oil][i]]
        files['all'].extend(files[oil])
    
    # write filenames to .yaml with timestamp ID
    now = datetime.now()
    dt_string = now.strftime("%d%m%Y_%H:%M:%S")
    out_f = output_dir+f'/MOHID_results_locations_{dt_string}.yaml'
    print(out_f)
    with open(out_f, 'w') as output_yaml:
        documents = yaml.safe_dump(files, output_yaml)
    
    return files
%%time
output_dir = '/home/rmueller/projects/def-allen/rmueller/MIDOSS/Visualization'
files = get_MOHID_netcdf_filenames(results_dir, output_dir)
files['akns'][0]
oil_types = [
        'akns', 
        'bunker', 
        'dilbit', 
        'jet', 
        'diesel', 
        'gas', 
        'other'
    ]
for oil in oil_types:
    print(f'{oil}: {len(files[oil])} completed runs')
from datetime import date

today = date.today()
print("Today's date:", today)
from datetime import datetime
# datetime object containing current date and time
now = datetime.now()
 
print("now =", now)

# dd/mm/YY H:M:S
dt_string = now.strftime("%d%m%Y_%H:%M:%S")
print("date and time =", dt_string)<jupyter_output>now = 2021-10-07 14:29:31.691427
date and time = 07102021_14:29:31
{'fraction_non_alphanumeric': 0.13200723327305605, 'fraction_numerical': 0.016952983725135623, 'mean_word_length': 3.4427710843373496, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '358954', 'license': 'Apache-2.0', 'n_tokens_mistral': 1688, 'n_tokens_neox': 1577, 'n_words': 388, 'path': 'notebooks/dev_get_MOHID_netcdf_filenames.ipynb', 'repo_name': 'MIDOSS/Visualization', 'size': '4424'}