<commit_before>#!/bin/bash
# <NAME>
#
# This scripts needs four or five arguments
#
# ${1} the first   argument is the MIP name
# ${2} the second  argument is the experiment name or MIP name in the latter case all MIP experiments are taken.
# ${3} the third   argument is the ec-earth model configuration
# ${4} the fourth  argument is the meta data template json file which is used as input, the file: resources/metadata-templates/cmip6-CMIP-piControl-metadata-template.json
#
#
# Run example:
#  ./modify-metadata-template.sh CMIP piControl EC-EARTH-CC cmip6-CMIP-piControl-metadata-template.json
#
# With this script it is possible to generate a dedicated metadata template json file for each ec-earth3 cmip6 MIP experiment.
#
# This script is part of the subpackage genecec (GENerate EC-Eearth Control output files)
# which is part of ece2cmor3.
#
# Note that this script is called in a loop over the MIP experiments by the script:
#  genecec.py

if [ "$#" -eq 4 ]; then
 mip=$1
 experiment=$2
 ececonf=$3
 input_template=$4


 if [ "${ececonf}" != 'EC-EARTH-AOGCM' ] && [ "${ececonf}" != 'EC-EARTH-HR' ] && [ "${ececonf}" != 'EC-EARTH-LR' ] && [ "${ececonf}" != 'EC-EARTH-CC' ] && [ "${ececonf}" != 'EC-EARTH-GrisIS' ] && [ "${ececonf}" != 'EC-EARTH-AerChem' ] && [ "${ececonf}" != 'EC-EARTH-Veg' ] && [ "${ececonf}" != 'EC-EARTH-Veg-LR' ]; then
  echo ' Error in ' $0 ': unkown ec-earth configuration: '  ${ececonf}
  exit
 fi

 if [ "${ececonf}" = 'EC-EARTH-AOGCM'   ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-HR'      ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-LR'      ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-CC'      ]; then declare -a model_components=('ifs' 'nemo' 'tm5' 'lpjg'); fi
 if [ "${ececonf}" = 'EC-EARTH-GrisIS'  ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-AerChem' ]; then declare -a model_components=('ifs' 'nemo' 'tm5'       ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg'     ]; then declare -a model_components=('ifs' 'nemo'       'lpjg'); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg-LR'  ]; then declare -a model_components=('ifs' 'nemo'       'lpjg'); fi

 #                    NAME IN SCRIPT                                 ECE CONF NAME       IFS RES     NEMO RES      TM5 RES                                  LPJG RES   PISCES RES  PISM RES    source_type
 if [ "${ececonf}" = 'EC-EARTH-AOGCM'   ]; then declare -a ece_res=('EC-Earth3'          'T255L91'  'ORCA1L75'    'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-HR'      ]; then declare -a ece_res=('EC-Earth3-HR'       'T511L91'  'ORCA025L75'  'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-LR'      ]; then declare -a ece_res=('EC-Earth3-LR'       'T159L91'  'ORCA1L75'    'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-CC'      ]; then declare -a ece_res=('EC-Earth3-CC'       'T255L91'  'ORCA1L75'    'native regular 2x3 degree latxlon grid'  'T255L91'  'ORCA1L75'  'none'      'AOGCM BGC'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-GrisIS'  ]; then declare -a ece_res=('EC-Earth3-GrIS'     'T255L91'  'ORCA1L75'    'none'                                    'none'     'none'      '5 x 5 km'  'AOGCM ISM'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-AerChem' ]; then declare -a ece_res=('EC-Earth3-AerChem'  'T255L91'  'ORCA1L75'    'native regular 2x3 degree latxlon grid'  'none'     'none'      'none'      'AOGCM AER CHEM' ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg'     ]; then declare -a ece_res=('EC-Earth3-Veg'      'T255L91'  'ORCA1L75'    'none'                                    'T255L91'  'none'      'none'      'AOGCM'          ); fi 
 if [ "${ececonf}" = 'EC-EARTH-Veg-LR'  ]; then declare -a ece_res=('EC-Earth3-Veg-LR'   'T159L91'  'ORCA1L75'    'none'                                    'T159L91'  'none'      'none'      'AOGCM'          ); fi 

 if [ "${ececonf}" = 'EC-EARTH-AOGCM'   ]; then declare -a nom_res=('EC-Earth3'          '100 km'   '50 km'       'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-HR'      ]; then declare -a nom_res=('EC-Earth3-HR'       '50 km'    '10 km'       'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-LR'      ]; then declare -a nom_res=('EC-Earth3-LR'       '100 km'   '50 km'       'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-CC'      ]; then declare -a nom_res=('EC-Earth3-CC'       '100 km'   '50 km'       '250 km'                                  '100 km'   '50 km'     'none'      'AOGCM BGC'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-GrisIS'  ]; then declare -a nom_res=('EC-Earth3-GrIS'     '100 km'   '50 km'       'none'                                    'none'     'none'      '5 km'      'AOGCM ISM'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-AerChem' ]; then declare -a nom_res=('EC-Earth3-AerChem'  '100 km'   '50 km'       '250 km'                                  'none'     'none'      'none'      'AOGCM AER CHEM' ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg'     ]; then declare -a nom_res=('EC-Earth3-Veg'      '100 km'   '50 km'       'none'                                    '100 km'   'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg-LR'  ]; then declare -a nom_res=('EC-Earth3-Veg-LR'   '100 km'   '50 km'       'none'                                    '100 km'   'none'      'none'      'AOGCM'          ); fi
 # https://www.earthsystemcog.org/site_media/projects/wip/CMIP6_global_attributes_filenames_CVs_v6.2.6.pdf
 # IFS  T511   T255   T159       ORCA1            ORCA0.25                TM5       LPJG=IFS
 #      40 km  80 km  125 km     0.67 * 111 km    0.25 * 0.67 * 111 km

 echo
 for i in "${model_components[@]}"
 do
    component="$i"

    echo ' Running ' $0 ${mip} ${experiment} ${ececonf} ${input_template} '  for ' ${component}

    if [ "${component}" != 'ifs' ] && [ "${component}" != 'nemo' ] && [ "${component}" != 'tm5' ] && [ "${component}" != 'lpjg' ]; then
     echo ' Error in ' $0 ': unkown ec-earth component: '  ${component} '  Valid options: ifs, nemo, tm5, or lpjg'
     exit
    fi

    if [ "${component}" = 'ifs' ]; then
     grid_label='gr'
    elif [ "${component}" = 'nemo' ]; then
     grid_label='gn'
    elif [ "${component}" = 'tm5' ]; then
     grid_label='gn'
    elif [ "${component}" = 'lpjg' ]; then
     grid_label='gr'
    fi

    output_template=metadata-cmip6-${mip}-${experiment}-${ececonf}-${component}-template.json

    # Creating and adjusting with sed the output meta data template json file:
    sed    's/"activity_id":                  "CMIP"/"activity_id":                  "'${mip}'"/' ${input_template} >       ${output_template}
    sed -i 's/"experiment_id":                "piControl"/"experiment_id":                "'${experiment}'"/'               ${output_template}
    sed -i 's/"source_id":                    "EC-Earth3"/"source_id":                    "'${ece_res[0]}'"/'               ${output_template}
    sed -i 's/"source":                       "EC-Earth3 (2019)"/"source":                       "'${ece_res[0]}'" (2019)/' ${output_template}  # The 2019 is correct as long no P verison from 2017 is taken.
    sed -i 's/"source_type":                  "AOGCM"/"source_type":                  "'"${ece_res[7]}"'"/'                 ${output_template}  # Note the double quote for the spaces in the variable
    sed -i 's/"grid":                         "T255L91"/"grid":                         "'${ece_res[1]}'"/'                 ${output_template}
    sed -i 's/"grid_label":                   "gr"/"grid_label":                   "'${grid_label}'"/'                      ${output_template}
    sed -i 's/"nominal_resolution":           "100 km"/"nominal_resolution":           "'"${nom_res[1]}"'"/'                ${output_template}

 done
 echo

else
    echo '  '
    echo '  This scripts requires  variable, e.g.:'
    echo '  ' $0 CMIP piControl EC-EARTH-AOGCM cmip6-CMIP-piControl-metadata-template.json
    echo '  '
fi<commit_msg>Add the use of all the model component resolution info + a bugfix for spaces in the ece_res array elements #214.<commit_after>#!/bin/bash
# <NAME>
#
# This scripts needs four or five arguments
#
# ${1} the first   argument is the MIP name
# ${2} the second  argument is the experiment name or MIP name in the latter case all MIP experiments are taken.
# ${3} the third   argument is the ec-earth model configuration
# ${4} the fourth  argument is the meta data template json file which is used as input, the file: resources/metadata-templates/cmip6-CMIP-piControl-metadata-template.json
#
#
# Run example:
#  ./modify-metadata-template.sh CMIP piControl EC-EARTH-CC cmip6-CMIP-piControl-metadata-template.json
#
# With this script it is possible to generate a dedicated metadata template json file for each ec-earth3 cmip6 MIP experiment.
#
# This script is part of the subpackage genecec (GENerate EC-Eearth Control output files)
# which is part of ece2cmor3.
#
# Note that this script is called in a loop over the MIP experiments by the script:
#  genecec.py

if [ "$#" -eq 4 ]; then
 mip=$1
 experiment=$2
 ececonf=$3
 input_template=$4


 if [ "${ececonf}" != 'EC-EARTH-AOGCM' ] && [ "${ececonf}" != 'EC-EARTH-HR' ] && [ "${ececonf}" != 'EC-EARTH-LR' ] && [ "${ececonf}" != 'EC-EARTH-CC' ] && [ "${ececonf}" != 'EC-EARTH-GrisIS' ] && [ "${ececonf}" != 'EC-EARTH-AerChem' ] && [ "${ececonf}" != 'EC-EARTH-Veg' ] && [ "${ececonf}" != 'EC-EARTH-Veg-LR' ]; then
  echo ' Error in ' $0 ': unkown ec-earth configuration: '  ${ececonf}
  exit
 fi

 if [ "${ececonf}" = 'EC-EARTH-AOGCM'   ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-HR'      ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-LR'      ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-CC'      ]; then declare -a model_components=('ifs' 'nemo' 'tm5' 'lpjg'); fi
 if [ "${ececonf}" = 'EC-EARTH-GrisIS'  ]; then declare -a model_components=('ifs' 'nemo'             ); fi
 if [ "${ececonf}" = 'EC-EARTH-AerChem' ]; then declare -a model_components=('ifs' 'nemo' 'tm5'       ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg'     ]; then declare -a model_components=('ifs' 'nemo'       'lpjg'); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg-LR'  ]; then declare -a model_components=('ifs' 'nemo'       'lpjg'); fi

 #                    NAME IN SCRIPT                                 ECE CONF NAME       IFS RES     NEMO RES      TM5 RES                                  LPJG RES   PISCES RES  PISM RES    source_type
 if [ "${ececonf}" = 'EC-EARTH-AOGCM'   ]; then declare -a ece_res=('EC-Earth3'          'T255L91'  'ORCA1L75'    'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-HR'      ]; then declare -a ece_res=('EC-Earth3-HR'       'T511L91'  'ORCA025L75'  'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-LR'      ]; then declare -a ece_res=('EC-Earth3-LR'       'T159L91'  'ORCA1L75'    'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-CC'      ]; then declare -a ece_res=('EC-Earth3-CC'       'T255L91'  'ORCA1L75'    'native regular 2x3 degree latxlon grid'  'T255L91'  'ORCA1L75'  'none'      'AOGCM BGC'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-GrisIS'  ]; then declare -a ece_res=('EC-Earth3-GrIS'     'T255L91'  'ORCA1L75'    'none'                                    'none'     'none'      '5 x 5 km'  'AOGCM ISM'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-AerChem' ]; then declare -a ece_res=('EC-Earth3-AerChem'  'T255L91'  'ORCA1L75'    'native regular 2x3 degree latxlon grid'  'none'     'none'      'none'      'AOGCM AER CHEM' ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg'     ]; then declare -a ece_res=('EC-Earth3-Veg'      'T255L91'  'ORCA1L75'    'none'                                    'T255L91'  'none'      'none'      'AOGCM'          ); fi 
 if [ "${ececonf}" = 'EC-EARTH-Veg-LR'  ]; then declare -a ece_res=('EC-Earth3-Veg-LR'   'T159L91'  'ORCA1L75'    'none'                                    'T159L91'  'none'      'none'      'AOGCM'          ); fi 

 if [ "${ececonf}" = 'EC-EARTH-AOGCM'   ]; then declare -a nom_res=('EC-Earth3'          '100 km'   '50 km'       'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-HR'      ]; then declare -a nom_res=('EC-Earth3-HR'       '50 km'    '10 km'       'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-LR'      ]; then declare -a nom_res=('EC-Earth3-LR'       '100 km'   '50 km'       'none'                                    'none'     'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-CC'      ]; then declare -a nom_res=('EC-Earth3-CC'       '100 km'   '50 km'       '250 km'                                  '100 km'   '50 km'     'none'      'AOGCM BGC'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-GrisIS'  ]; then declare -a nom_res=('EC-Earth3-GrIS'     '100 km'   '50 km'       'none'                                    'none'     'none'      '5 km'      'AOGCM ISM'      ); fi
 if [ "${ececonf}" = 'EC-EARTH-AerChem' ]; then declare -a nom_res=('EC-Earth3-AerChem'  '100 km'   '50 km'       '250 km'                                  'none'     'none'      'none'      'AOGCM AER CHEM' ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg'     ]; then declare -a nom_res=('EC-Earth3-Veg'      '100 km'   '50 km'       'none'                                    '100 km'   'none'      'none'      'AOGCM'          ); fi
 if [ "${ececonf}" = 'EC-EARTH-Veg-LR'  ]; then declare -a nom_res=('EC-Earth3-Veg-LR'   '100 km'   '50 km'       'none'                                    '100 km'   'none'      'none'      'AOGCM'          ); fi
 # https://www.earthsystemcog.org/site_media/projects/wip/CMIP6_global_attributes_filenames_CVs_v6.2.6.pdf
 # IFS  T511   T255   T159       ORCA1            ORCA0.25                TM5       LPJG=IFS
 #      40 km  80 km  125 km     0.67 * 111 km    0.25 * 0.67 * 111 km

 echo
 for i in "${model_components[@]}"
 do
    component="$i"

    echo ' Running ' $0 ${mip} ${experiment} ${ececonf} ${input_template} '  for ' ${component}

    if [ "${component}" != 'ifs' ] && [ "${component}" != 'nemo' ] && [ "${component}" != 'tm5' ] && [ "${component}" != 'lpjg' ]; then
     echo ' Error in ' $0 ': unkown ec-earth component: '  ${component} '  Valid options: ifs, nemo, tm5, or lpjg'
     exit
    fi

    if [ "${component}" = 'ifs' ]; then
     grid_label='gr'
     res_index=1
    elif [ "${component}" = 'nemo' ]; then
     grid_label='gn'
     res_index=2
    elif [ "${component}" = 'tm5' ]; then
     grid_label='gn'
     res_index=3
    elif [ "${component}" = 'lpjg' ]; then
     grid_label='gr'
     res_index=4
    fi

    output_template=metadata-cmip6-${mip}-${experiment}-${ececonf}-${component}-template.json

    # Creating and adjusting with sed the output meta data template json file:
    sed    's/"activity_id":                  "CMIP"/"activity_id":                  "'${mip}'"/' ${input_template} >       ${output_template}
    sed -i 's/"experiment_id":                "piControl"/"experiment_id":                "'${experiment}'"/'               ${output_template}
    sed -i 's/"source_id":                    "EC-Earth3"/"source_id":                    "'${ece_res[0]}'"/'               ${output_template}
    sed -i 's/"source":                       "EC-Earth3 (2019)"/"source":                       "'${ece_res[0]}'" (2019)/' ${output_template}  # The 2019 is correct as long no P verison from 2017 is taken.
    sed -i 's/"source_type":                  "AOGCM"/"source_type":                  "'"${ece_res[7]}"'"/'                 ${output_template}  # Note the double quote for the spaces in the variable
    sed -i 's/"grid_label":                   "gr"/"grid_label":                   "'${grid_label}'"/'                      ${output_template}
    sed -i 's/"grid":                         "T255L91"/"grid":                         "'"${ece_res[${res_index}]}"'"/'    ${output_template}
    sed -i 's/"nominal_resolution":           "100 km"/"nominal_resolution":           "'"${nom_res[${res_index}]}"'"/'     ${output_template}

 done
 echo

else
    echo '  '
    echo '  This scripts requires  variable, e.g.:'
    echo '  ' $0 CMIP piControl EC-EARTH-AOGCM cmip6-CMIP-piControl-metadata-template.json
    echo '  '
fi
{'fraction_non_alphanumeric': 0.1663475485929794, 'fraction_numerical': 0.03057731360603423, 'mean_word_length': 1.8470432771721177, 'pattern_counts': {'":': 32, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '258007', 'n_tokens_mistral': 6455, 'n_tokens_neox': 5942, 'n_words': 1490}