<commit_before>#!/usr/bin/env bash
# <NAME>
#
# This script converts a component json file produced by genecec including the preferences
# into a flat json without the component structue (ifs, nemo, lpjg & tm5) such that it can be
# read with: checkvars --drq flat-json-file.json
#
# Note that when a flat json file is given (instead of a component json file) the produced json
# file will be just equal to the input flat json file.
#
# Run this script without arguments for examples how to call this script.
#

if [ "$#" -eq 1 ]; then

 input_json_file=$1
 output_json_file=${input_json_file##*/}
 output_json_file=${output_json_file/.json/-flat.json}

 # Convert vars json to drq json:
 sed -e '/"ifs":/d' -e '/"nemo":/d' -e '/"lpjg":/d' -e '/"tm5":/d' -e '/    },$/d' -e 's/        ]$/        ],/' ${input_json_file} | head --lines=-2 >  ${output_json_file}
 echo '        ]'                                                                                                                                     >> ${output_json_file}
 echo '}'                                                                                                                                             >> ${output_json_file}

 echo
 echo ' The script' $0 ' produced the file;'
 echo '  ' ${output_json_file}
 echo

else
 echo
 echo '  This scripts requires one argument, a json file, e.g.:'
 echo '  ' $0 knmi23-dutch-scenarios/historical-EC-EARTH-AOGCM-plev23r/cmip6-data-request-varlist-CMIP-historical-EC-EARTH-AOGCM.json
 echo
fi<commit_msg>Add a fix for a json syntax error in case the tm5 list is not empty #701.<commit_after>#!/usr/bin/env bash
# <NAME>
#
# This script converts a component json file produced by genecec including the preferences
# into a flat json without the component structue (ifs, nemo, lpjg & tm5) such that it can be
# read with: checkvars --drq flat-json-file.json
#
# Note that when a flat json file is given (instead of a component json file) the produced json
# file will be just equal to the input flat json file.
#
# Run this script without arguments for examples how to call this script.
#

if [ "$#" -eq 1 ]; then

 input_json_file=$1
 output_json_file=${input_json_file##*/}
 output_json_file=${output_json_file/.json/-flat.json}

 # Convert vars json to drq json:
 sed -e '/"ifs":/d' -e '/"nemo":/d' -e '/"lpjg":/d' -e '/"tm5":/d' -e '/    },$/d' -e 's/        ]$/        ],/' ${input_json_file} | head --lines=-2 >  ${output_json_file}
 echo '        ]'                                                                                                                                     >> ${output_json_file}
 echo '}'                                                                                                                                             >> ${output_json_file}

 # Check wheter the two but last line (in case the tm5 list is not empty) contains an invalid json line:
 check_end=`tail -n3 ${output_json_file} | head -n1`
 if [ "${check_end}" == '        ],' ]; then
  head --lines=-3 ${output_json_file} >  ${output_json_file}.tmp
  echo '        ]'                    >> ${output_json_file}.tmp
  echo '}'                            >> ${output_json_file}.tmp
  mv -f ${output_json_file}.tmp ${output_json_file}
 fi

 echo
 echo ' The script' $0 ' produced the file;'
 echo '  ' ${output_json_file}
 echo

else
 echo
 echo '  This scripts requires one argument, a json file, e.g.:'
 echo '  ' $0 knmi23-dutch-scenarios/historical-EC-EARTH-AOGCM-plev23r/cmip6-data-request-varlist-CMIP-historical-EC-EARTH-AOGCM.json
 echo
fi
{'fraction_non_alphanumeric': 0.12020173718128327, 'fraction_numerical': 0.008966096945923228, 'mean_word_length': 1.837837837837838, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2492662', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1038, 'n_words': 338}