<commit_before># Reply from <NAME> at 16 Jul 2018, 12:43 dreq: looping over MIPs and experiments
#
# There is some code in the attached script which does the looping you ask for. There are two versions: (1) looping over experiments defined by each MIP and (2) looping over experiments from which each MIP requests data (generally a larger set). I'm working on some support methods which will simplify the code for (2)
# 

import json
import logging
import os

from ece2cmor3 import components
from ece2cmor3 import ece2cmorlib, cmor_source, cmor_target, cmor_task
from ece2cmor3.cmor_source import create_cmor_source

log = logging.getLogger(__name__)

json_source_key = "source"
json_target_key = "target"
json_table_key = "table"
json_mask_key = "mask"
json_masked_key = "masked"

omit_vars_file_01 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-01.xlsx")
omit_vars_file_02 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-02.xlsx")
omit_vars_file_03 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-03.xlsx")
omit_vars_file_04 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-04.xlsx")
omit_vars_file_05 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-05.xlsx")<commit_msg>Remove the add of a totally accidentally added block of comment in commit a98ec313989063fb01a39124ffd09bb22282274c<commit_after>import json
import logging
import os

from ece2cmor3 import components
from ece2cmor3 import ece2cmorlib, cmor_source, cmor_target, cmor_task
from ece2cmor3.cmor_source import create_cmor_source

log = logging.getLogger(__name__)

json_source_key = "source"
json_target_key = "target"
json_table_key = "table"
json_mask_key = "mask"
json_masked_key = "masked"

omit_vars_file_01 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-01.xlsx")
omit_vars_file_02 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-02.xlsx")
omit_vars_file_03 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-03.xlsx")
omit_vars_file_04 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-04.xlsx")
omit_vars_file_05 = os.path.join(os.path.dirname(__file__), "resources/lists-of-omitted-variables",
                                 "list-of-omitted-variables-05.xlsx")
{'fraction_non_alphanumeric': 0.10496311200536552, 'fraction_numerical': 0.031857813547954396, 'mean_word_length': 3.3294629898403483, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1515018', 'n_tokens_mistral': 1031, 'n_tokens_neox': 971, 'n_words': 188}