<jupyter_start><jupyter_text>*** **FloodProofs Labs - HMC Training - Model Runner - Realtime Execution** In the laboratory of **Continuum Model Runner - Realtime Execution** we will perform the following points:   * Configure the libraries and the dependecies of the laboratory;   * Read the configuration file of the laboratory;   * Initialize the instance of the Continuum model --> Configure the algorithm and the ancillary datasets;   * Build the instance of the Continumm model --> Configure the static and dynamic datasets;   * Run the instance of the Continuum model --> Execute the Continuum model    * Finalize the instance of the Continuum model --> Organize and dump the outcome datasets.      **Import libraries and dependencies**<jupyter_code>%matplotlib inline
%matplotlib widget
# Add hmc library to the paths environment
import sys
import os

# Info
print(' ==> Add HMC library in the environment ... ') 

folder_home_env = os.path.expanduser('~')
folder_library_hmc = "/fp_labs_hmc/library/jupyter_hmc/"

folder_library_hmc = folder_home_env + folder_library_hmc
sys.path.append(folder_library_hmc)

# Info
print(' ===> Added in: ' + folder_library_hmc)
print(' ==> Add HMC library in the environment ... DONE')  
# Libraries
print(' ==> Load generic and package libraries ... ')

import pickle
import argparse
import logging
import time
import pandas as pd

# Import method to adapt settings file
from library.jupyter_generic.lib_jupyter_data_io_generic import define_file_path, define_file_template, \
    fill_file_template, create_dframe_ts, get_path_root, get_path_folders, get_folders_time

from library.jupyter_generic.lib_jupyter_utils_io import update_json_file
from library.jupyter_generic.lib_jupyter_data_io_json import read_file_settings

# Import coupler and driver classes
from hmc.driver.configuration.drv_configuration_hmc_logging import ModelLogging
from hmc.coupler.cpl_hmc_manager import ModelInitializer, ModelCleaner
from hmc.coupler.cpl_hmc_builder import ModelBuilder
from hmc.coupler.cpl_hmc_runner import ModelRunner
from hmc.coupler.cpl_hmc_finalizer import ModelFinalizer

import ipywidgets as widgets
from ipywidgets import interact, interact_manual

def write_pickle(file_name, file_data):
    if os.path.exists(file_name):
        os.remove(file_name)
    with open(file_name,'wb') as file_handle:
        pickle.dump(file_data, file_handle) 

def read_pickle(file_name):
    if not os.path.exists(file_name):
        raise IOError('File ' + file_name + ' not found!')
    file_handle = open(file_name,'rb')  
    file_data = pickle.load(file_handle)  
    return file_data

# File name for sharing variable(s)
file_name_settings = "fp_labs_runner_hmc_realtime_execution.json"
file_name_shared_vars = "shared_variables.workspace"

# Info
print(' ==> Load generic and package libraries ... DONE')<jupyter_output>==> Load generic and package libraries ... 
 ==> Load generic and package libraries ... DONE<jupyter_text>**Configure the Continuum laboratory**- Set the notebook arguments:<jupyter_code># Define filename by template definition(s)
print(' ==> Define settings args ... ')

# Read data from settings algorithm file
settings_info = read_file_settings(file_name_settings)

info_time_run = settings_info['info']['time_run']
info_domain = settings_info['info']['domain_name']
info_run_name = settings_info['info']['run_name']

script_settings_default = define_file_path(settings_info['configuration_info']['default'])
script_settings_defined = define_file_path(settings_info['configuration_info']['defined'])

script_datasets_defined = define_file_path(settings_info['datasets_info'])

# Info
print(' ===> TimeRun: ' + str(info_time_run))
print(' ===> DomainName: ' + info_domain)
print(' ===> RunName: ' + info_run_name)
print(' ==> Define settings args ... DONE')
# Get static and dynamic filename(s)
file_path_section_shp = script_datasets_defined['data_static']
file_path_run_ts_img = script_datasets_defined['run_realtime_ts_images']
file_path_run_ts_hydro = script_datasets_defined['run_realtime_ts_hydrograph']

# Get data dynamic root folder
folder_name_dynamic_root = get_path_folders(script_datasets_defined['data_dynamic'])
# Define list of available folders and time(s)
time_run_available = get_folders_time(folder_name_dynamic_root)
# Display available time run
time_run_selection = widgets.Select(options=time_run_available, description='Time Run', disabled=False)
display(time_run_selection)  
# Parser the selection from the scroll menu and define the time as a timestamp object
info_time_run = pd.Timestamp(time_run_selection.value)

# Info time run
print(' ==> Time Run: ' + str(info_time_run) )
# Define filename by template definition(s)
print(' ==> Define folder and file name ... ')

file_template_filled = define_file_template(
    info_time_run, run_name=info_run_name, domain_name=info_domain, template_default=settings_info['template'])

script_settings_defined['settings_algorithm'] = fill_file_template(
    script_settings_defined['settings_algorithm'],
    template_filled=file_template_filled, template_default=settings_info['template'])

script_settings_defined['settings_datasets'] = fill_file_template(
    script_settings_defined['settings_datasets'],
    template_filled=file_template_filled, template_default=settings_info['template'])

file_path_run_ts_img = fill_file_template(
    file_path_run_ts_img, template_filled=file_template_filled, template_default=settings_info['template'])
file_path_run_ts_hydro = fill_file_template(
    file_path_run_ts_hydro, template_filled=file_template_filled, template_default=settings_info['template'])

# Info
print(' ===> Filename Section: ' + file_path_section_shp)
print(' ===> Filename Hydrograph: ' + file_path_run_ts_hydro)
print(' ===> Filename Images: ' + file_path_run_ts_img)
print(' ==> Define folder and file name ... DONE')<jupyter_output>==> Define folder and file name ... 
 ===> Filename Section: /home/fabio/fp_labs_datasets/data_static/shapefile/fp_sections_marche.shp
 ===> Filename Hydrograph: /home/fabio/fp_run_hmc/jupyter_ws/outcome/timeseries/hmc.hydrograph.txt
 ===> Filename Images: /home/fabio/fp_run_hmc/jupyter_ws/outcome/images/hmc_ts_group{group_name}_{time_name}.png
 ==> Define folder and file name ... DONE<jupyter_text>- Update the configuration file(s) according with the environment variable(s)<jupyter_code># Info
print(' ==> Update configuration file(s) ... ')

# Set environment variables to json settings file(s)
update_json_file(script_settings_default['settings_algorithm'], script_settings_defined['settings_algorithm'],
                 update_dict={'$ENV_HOME': folder_home_env, 
                              '$ENV_TIME': info_time_run.strftime("%Y%m%d_%H"), '$ENV_RUN_NAME': info_run_name})
update_json_file(script_settings_default['settings_datasets'], script_settings_defined['settings_datasets'],
                 update_dict={'$ENV_HOME': folder_home_env, 
                              '$ENV_TIME': info_time_run.strftime("%Y%m%d_%H"), '$ENV_RUN_NAME': info_run_name})

script_settings_algorithm_defined = script_settings_defined['settings_algorithm']
script_settings_datasets_defined = script_settings_defined['settings_datasets']

# Info
print(' ===> Settings algorithm file: ' + script_settings_algorithm_defined)
print(' ===> Settings datasets file: ' + script_settings_datasets_defined)
print(' ==> Update configuration file(s) ... DONE')<jupyter_output>==> Update configuration file(s) ... 
 ===> Settings algorithm file: /home/fabio/fp_labs_hmc/fp_labs_runner_hmc_algorithm_202101220600.json
 ===> Settings datasets file: /home/fabio/fp_labs_hmc/fp_labs_runner_hmc_datasets_202101220600.json
 ==> Update configuration file(s) ... DONE<jupyter_text>**Initialize the instance of the Continuum model**- Set the logging file:<jupyter_code># Info
print(' ==> Set logging file ... ')

# Set logging file
driver_hmc_logging = ModelLogging(script_settings_algorithm_defined)
log_stream = driver_hmc_logging.configure_logging()

# Set the verbose debug level
log_stream.setLevel(logging.WARNING)

# Info
print(' ==> Set logging file ... DONE')<jupyter_output>==> Set logging file ... 
 ==> Set logging file ... DONE<jupyter_text>- Configure the model initializer class<jupyter_code># Info
print(' ==> Configure the model initializer class ... ')

# Set the verbose debug level
log_stream.setLevel(logging.ERROR)

# Configure model initializer class
driver_hmc_initializer = ModelInitializer(file_algorithm=script_settings_algorithm_defined,
                                          file_datasets=script_settings_datasets_defined, time=info_time_run)

# Info
print(' ==> Configure the model initializer class ... DONE')<jupyter_output>==> Configure the model initializer class ... 
 ==> Configure the model initializer class ... DONE<jupyter_text>- Configure the algorithm and the ancillary datasets<jupyter_code># Configure algorithm
time_series_collections, time_info_collections, \
    run_info_collections, run_cline_collections = driver_hmc_initializer.configure_algorithm()
# Configure ancillary datasets
ancillary_datasets_collections = driver_hmc_initializer.configure_ancillary_datasets(time_info_collections)<jupyter_output><empty_output><jupyter_text>**Build the instance of the Continuum model**- Configure the model builder class<jupyter_code># Info
print(' ==> Configure the model builder class ... ')

# Set the verbose debug level
log_stream.setLevel(logging.WARNING)

# Configure model builder class
driver_hmc_builder = ModelBuilder(
    obj_geo_reference=driver_hmc_initializer.dset_ref_geo,
    obj_args=driver_hmc_initializer.obj_args,
    obj_run=driver_hmc_initializer.obj_run,
    obj_ancillary=driver_hmc_initializer.obj_ancillary)

# Info
print(' ==> Configure the model builder class ... DONE')<jupyter_output>==> Configure the model builder class ... 
 ==> Configure the model builder class ... DONE<jupyter_text>- Dump variable to initialize the notebook for realtime plot(s)<jupyter_code># Info
print(' ==> Dump shared variables ... ')

# Dump variable to pickle ==> TO SET THE SECOND NOTEBOOK
variables_shared={'time_series_collections': time_series_collections, 
                  'time_run': info_time_run, 'run_name': info_run_name,
                  'file_name_section': file_path_section_shp, 'file_name_hydrograph': file_path_run_ts_hydro,
                  'file_name_images': file_path_run_ts_img,
                  'driver_hmc_builder': driver_hmc_builder}
write_pickle(file_name_shared_vars, variables_shared)

# Info
print(' ==> Dump shared variables ... DONE')<jupyter_output>==> Dump shared variables ... 
 ==> Dump shared variables ... DONE<jupyter_text>- Organize the static and the dynamic datasets<jupyter_code># Info
print(' ==> Organize the model static datasets ... ')

# Set the verbose debug level
log_stream.setLevel(logging.ERROR)

# Configure static datasets
static_datasets_collections = driver_hmc_builder.configure_static_datasets(ancillary_datasets_collections)

# Info
print(' ==> Organize the model static datasets ... DONE')
# Info
print(' ==> Organize the model dynamic datasets ... ')

# Set the verbose debug level
log_stream.setLevel(logging.INFO)

# Configure dynamic datasets
forcing_datasets_collections = driver_hmc_builder.configure_dynamic_datasets(
    time_series_collections, time_info_collections, static_datasets_collections, ancillary_datasets_collections)

# Info
print(' ==> Organize the model dynamic datasets ... DONE')<jupyter_output>2021-01-29 17:03:23,293 hmc_logger   INFO      #### Configure dynamic source datasets ...                                      cpl_hmc_builder.py:[112    - configure_dynamic_datasets()] 
2021-01-29 17:03:23,298 hmc_logger   INFO      ----> Organize dynamic datasets ...                                             drv_dataset_hmc_base_source.py:[459    - organize_data_dynamic()] 
2021-01-29 17:03:23,299 hmc_logger   INFO      -----> Run deterministic ...                                                    drv_dataset_hmc_base_source.py:[483    - organize_data_dynamic()] 
2021-01-29 17:03:23,301 hmc_logger   INFO      ------> Collect Gridded source filename(s) ...                                  drv_dataset_hmc_io_dynamic_forcing.py:[936    - collect_filename    ()] 
2021-01-29 17:03:23,303 hmc_logger   INFO      -------> Type ARCHIVE ...                                                       drv_dataset_hmc_io_dynamic_forcing.py:[945    - collect_filename    ()] 
2021-01-29 17:03:23,305 h[...]<jupyter_text>**Run the instance of the Continuum model**- Configure the runner model class<jupyter_code># Info
print(' ==> Configure the model runner class ... ')

# Set the verbose debug level
log_stream.setLevel(logging.ERROR)

# Configure model runner class
driver_hmc_runner = ModelRunner(time_info=time_info_collections, run_info=run_info_collections,
                                command_line_info=run_cline_collections,
                                obj_args=driver_hmc_initializer.obj_args,
                                obj_ancillary=driver_hmc_initializer.obj_ancillary)

# Info
print(' ==> Configure the model runner class ... DONE')<jupyter_output>==> Configure the model runner class ... 
 ==> Configure the model runner class ... DONE<jupyter_text>- Execute the instance of the model<jupyter_code># Info
print(' ==> Execute the model run ... ')

# Set the verbose debug level
log_stream.setLevel(logging.INFO)

# Configure the model execution
driver_hmc_runner.configure_execution(ancillary_datasets_collections)

# Info
print(' ==> Execute the model run ... DONE')<jupyter_output>2021-01-29 17:08:43,484 hmc_logger   INFO      #### Configure execution ...                                                    cpl_hmc_runner.py:[107    - configure_execution ()] 
2021-01-29 17:08:43,486 hmc_logger   INFO      ------> Process execution: /home/fabio/fp_run_hmc/jupyter_ws/exec/HMC_Model_V3_jupyter_ws.x /home/fabio/fp_run_hmc/jupyter_ws/exec/marche.info.txt ...  lib_utils_process.py:[42     - exec_process        ()]<jupyter_text>**Finalize the instance of the Continuum model**- Configure the finalizer class of the model<jupyter_code># Info
print(' ==> Configure the model finalizer class ... ')

# Set the verbose debug level
log_stream.setLevel(logging.ERROR)

# Configure model finalizer class
driver_hmc_finalizer = ModelFinalizer(
    collection_dynamic=forcing_datasets_collections,
    obj_geo_reference=driver_hmc_initializer.dset_ref_geo,
    obj_args=driver_hmc_initializer.obj_args,
    obj_run=driver_hmc_initializer.obj_run,
    obj_ancillary=driver_hmc_initializer.obj_ancillary)

# Info
print(' ==> Configure the model finalizer class ... DONE')<jupyter_output>==> Configure the model finalizer class ... 
 ==> Configure the model finalizer class ... DONE<jupyter_text>- Dump the model results in gridded, point and time-series format<jupyter_code># Info
print(' ==> Dump the model outcome datasets ... ')

# Set the verbose debug level
log_stream.setLevel(logging.ERROR)

# Configure the outcome datasets
outcome_datasets_collections = driver_hmc_finalizer.configure_dynamic_datasets(
    time_series_collections, time_info_collections, static_datasets_collections, ancillary_datasets_collections)

# Info
print(' ==> Dump the model outcome datasets ... DONE')<jupyter_output>==> Dump the model outcome datasets ... 
 ==> Dump the model outcome datasets ... DONE<jupyter_text>- Dump the model results in collections format<jupyter_code># Info
print(' ==> Dump the model summary datasets ... ')

# Set the verbose debug level
log_stream.setLevel(logging.ERROR)

# Configure the summary datasets
driver_hmc_finalizer.configure_summary_datasets(
    time_series_collections, time_info_collections, static_datasets_collections, outcome_datasets_collections)

# Info
print(' ==> Dump the model summary datasets ... DONE')<jupyter_output>==> Dump the model summary datasets ... 
 ==> Dump the model summary datasets ... DONE
{'fraction_non_alphanumeric': 0.09986842929641, 'fraction_numerical': 0.011340141595138149, 'mean_word_length': 4.885693215339233, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '319399', 'license': 'MIT', 'n_tokens_mistral': 4996, 'n_tokens_neox': 4839, 'n_words': 1110, 'path': 'fp_labs_runner_hmc_realtime_execution.ipynb', 'repo_name': 'c-hydro/fp-labs-hmc', 'size': '15961'}