<commit_before>''' Home of the configuration defaults.

'''

import collections
import pprint
import logging

logger = logging.getLogger(__name__)

def pp_config (config):
    ''' Pretty print the config, this should be part of the
    default logging to the debug logs.

    Args:
        - config (dict) : The config data structure
    '''

    logger.debug(pprint.pformat(config, indent=2))
    return

def recursive_update (template, userdata):
    ''' Recursively update the template with userdata.
    If we don't do this the value updates for nested collections
    would get simply overwritten rathen than updated.

    Args:
        template (dict) : The templated dict
        userdata (dict) : User supplied config dict structure

    Returns:
        Updated template stucture.
    '''

    for k, v in userdata.items():
        if isinstance (v, collections.Mapping):
            template[k] = recursive_update(template.get(k, {}), v)
        else:
            template[k] = v

    return template

def update_config (config, rundir):
    ''' Update the config datastructure with defaults. This is the one centralized
    location where the default live.

    Args:
         - config (dict) : The config dictionary

    Returns:
         - A standardized config dict if a config was passed, else None.
    '''

    if not config :
        return None

    config_base = { "sites" : [],
                    "globals" : {
                        "lazyErrors" : False
                    },
                    "controller" : {
                        "mode" : "auto"
                    }
    }

    sites = config["sites"] ; del config["sites"]

    config_base.update(config)

    config_sites = []
    for site in sites:
        site_base = {
            "auth" : {},
            "execution" : {
                "scriptDir" : rundir,
                "block" : {
                    "nodes" : 1,
                    "taskBlocks" : 1,
                    "minBlocks"  : 0,
                    "initBlocks" : 0,
                    "maxBlocks"  : 10,
                    "parallelism" : 0.75,
                    "walltime"   : "00:20:00",
                    "options" : {
                    }
                }
            }
        }

        recursive_update(site_base, site)
        config_sites.extend([site_base])

    config_base["sites"] = config_sites

    pp_config(config_base)

    return config_base
<commit_msg>Adding more defaults<commit_after>''' Home of the configuration defaults.

'''

import collections
import pprint
import logging

logger = logging.getLogger(__name__)

def pp_config (config):
    ''' Pretty print the config, this should be part of the
    default logging to the debug logs.

    Args:
        - config (dict) : The config data structure
    '''

    logger.debug(pprint.pformat(config, indent=2))
    return

def recursive_update (template, userdata):
    ''' Recursively update the template with userdata.
    If we don't do this the value updates for nested collections
    would get simply overwritten rathen than updated.

    Args:
        template (dict) : The templated dict
        userdata (dict) : User supplied config dict structure

    Returns:
        Updated template stucture.
    '''

    for k, v in userdata.items():
        if isinstance (v, collections.Mapping):
            template[k] = recursive_update(template.get(k, {}), v)
        else:
            template[k] = v

    return template

def update_config (config, rundir):
    ''' Update the config datastructure with defaults. This is the one centralized
    location where the default live.

    Args:
         - config (dict) : The config dictionary

    Returns:
         - A standardized config dict if a config was passed, else None.
    '''

    if not config :
        return None

    config_base = { "sites" : [],
                    "globals" : {
                        "lazyErrors" : False,
                        "usageTracking" : True
                    },
                    "controller" : {
                        "mode" : "auto"
                    }
    }

    sites = config["sites"] ; del config["sites"]

    config_base.update(config)

    config_sites = []
    for site in sites:
        site_base = {
            "auth" : {},
            "execution" : {
                "scriptDir" : rundir,
                "block" : {
                    "nodes" : 1,
                    "taskBlocks" : 1,
                    "minBlocks"  : 0,
                    "initBlocks" : 0,
                    "maxBlocks"  : 10,
                    "parallelism" : 0.75,
                    "walltime"   : "00:20:00",
                    "options" : {
                    }
                }
            }
        }

        recursive_update(site_base, site)
        config_sites.extend([site_base])

    config_base["sites"] = config_sites

    pp_config(config_base)

    return config_base

{'fraction_non_alphanumeric': 0.09369296288785237, 'fraction_numerical': 0.006489555871020077, 'mean_word_length': 1.5849056603773586, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5429397', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1268, 'n_words': 394}