<commit_before>#
# Copyright (C) 2012 - 2015 <NAME> <<EMAIL>>
# License: MIT
#
# pylint: disable=unused-import,import-error,invalid-name
"""Public APIs of anyconfig module.
"""
from __future__ import absolute_import

from anyconfig.globals import LOGGER
import anyconfig.backends
import anyconfig.backend.json
import anyconfig.compat
import anyconfig.mergeabledict
import anyconfig.parser
import anyconfig.template
import anyconfig.utils

# Import some global constants will be re-exported:
from anyconfig.mergeabledict import (
    MS_REPLACE, MS_NO_REPLACE, MS_DICTS, MS_DICTS_AND_LISTS, MERGE_STRATEGIES,
    get, set_  # flake8: noqa
)
from anyconfig.parser import PATH_SEPS
from anyconfig.schema import validate, gen_schema

# Re-export and aliases:
list_types = anyconfig.backends.list_types  # flake8: noqa
container = anyconfig.mergeabledict.MergeableDict


def _validate(config, schema, format_checker=None):
    """
    Wrapper function for anyconfig.schema.vaildate.

    :param config: Configuration object :: container
    :param schema: JSON schema object :: container
    :param format_checker: A format property checker object of which class is
        inherited from jsonschema.FormatChecker, it's default if None given.

    :return: True if validation suceeds or jsonschema module is not available.
    """
    (ret, msg) = validate(config, schema, format_checker, True)
    if msg:
        LOGGER.warn(msg)

    return ret


def set_loglevel(level):
    """
    :param level: Log level, e.g. logging.INFO and logging.WARN.
    """
    LOGGER.setLevel(level)


def find_loader(config_path, forced_type=None):
    """
    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type

    :return: Parser-inherited class object
    """
    if forced_type is not None:
        cparser = anyconfig.backends.find_by_type(forced_type)
        if not cparser:
            LOGGER.error("No parser found for given type: %s", forced_type)
            return None
    else:
        cparser = anyconfig.backends.find_by_file(config_path)
        if not cparser:
            LOGGER.error("No parser found for given file: %s", config_path)
            return None

    LOGGER.debug("Using config parser of type: %s", cparser.type())
    return cparser


def single_load(config_path, forced_type=None, ignore_missing=False,
                ac_template=False, ac_context=None, ac_schema=None,
                **kwargs):
    """
    Load single config file.

    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type
    :param ignore_missing: Ignore and just return empty result if given file
        (``config_path``) does not exist
    :param ac_template: Assume configuration file may be a template file and
        try to compile it AAR if True
    :param ac_context: A dict presents context to instantiate template
    :param ac_schema: JSON schema file path to validate given config file
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend

    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    config_path = anyconfig.utils.ensure_expandusr(config_path)

    cparser = find_loader(config_path, forced_type)
    if cparser is None:
        return None

    if ac_schema is not None:
        kwargs["ac_schema"] = None  # Avoid infinit loop
        format_checker = kwargs.get("format_checker", None)
        schema = load(ac_schema, forced_type=forced_type,
                      ignore_missing=ignore_missing, ac_template=ac_template,
                      ac_context=ac_context, **kwargs)

    LOGGER.info("Loading: %s", config_path)
    if ac_template:
        try:
            LOGGER.debug("Compiling: %s", config_path)
            config_content = anyconfig.template.render(config_path, ac_context)
            config = cparser.loads(config_content,
                                   ignore_missing=ignore_missing, **kwargs)
            if ac_schema is not None:
                if not _validate(config, schema, format_checker):
                    return None

            return config

        except Exception as exc:
            LOGGER.debug("Exc=%s", str(exc))
            LOGGER.warn("Failed to compile %s, fallback to no template "
                        "mode", config_path)

    config = cparser.load(config_path, ignore_missing=ignore_missing, **kwargs)
<commit_msg>add log message when loading schema file<commit_after>#
# Copyright (C) 2012 - 2015 <NAME> <<EMAIL>>
# License: MIT
#
# pylint: disable=unused-import,import-error,invalid-name
"""Public APIs of anyconfig module.
"""
from __future__ import absolute_import

from anyconfig.globals import LOGGER
import anyconfig.backends
import anyconfig.backend.json
import anyconfig.compat
import anyconfig.mergeabledict
import anyconfig.parser
import anyconfig.template
import anyconfig.utils

# Import some global constants will be re-exported:
from anyconfig.mergeabledict import (
    MS_REPLACE, MS_NO_REPLACE, MS_DICTS, MS_DICTS_AND_LISTS, MERGE_STRATEGIES,
    get, set_  # flake8: noqa
)
from anyconfig.parser import PATH_SEPS
from anyconfig.schema import validate, gen_schema

# Re-export and aliases:
list_types = anyconfig.backends.list_types  # flake8: noqa
container = anyconfig.mergeabledict.MergeableDict


def _validate(config, schema, format_checker=None):
    """
    Wrapper function for anyconfig.schema.vaildate.

    :param config: Configuration object :: container
    :param schema: JSON schema object :: container
    :param format_checker: A format property checker object of which class is
        inherited from jsonschema.FormatChecker, it's default if None given.

    :return: True if validation suceeds or jsonschema module is not available.
    """
    (ret, msg) = validate(config, schema, format_checker, True)
    if msg:
        LOGGER.warn(msg)

    return ret


def set_loglevel(level):
    """
    :param level: Log level, e.g. logging.INFO and logging.WARN.
    """
    LOGGER.setLevel(level)


def find_loader(config_path, forced_type=None):
    """
    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type

    :return: Parser-inherited class object
    """
    if forced_type is not None:
        cparser = anyconfig.backends.find_by_type(forced_type)
        if not cparser:
            LOGGER.error("No parser found for given type: %s", forced_type)
            return None
    else:
        cparser = anyconfig.backends.find_by_file(config_path)
        if not cparser:
            LOGGER.error("No parser found for given file: %s", config_path)
            return None

    LOGGER.debug("Using config parser of type: %s", cparser.type())
    return cparser


def single_load(config_path, forced_type=None, ignore_missing=False,
                ac_template=False, ac_context=None, ac_schema=None,
                **kwargs):
    """
    Load single config file.

    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type
    :param ignore_missing: Ignore and just return empty result if given file
        (``config_path``) does not exist
    :param ac_template: Assume configuration file may be a template file and
        try to compile it AAR if True
    :param ac_context: A dict presents context to instantiate template
    :param ac_schema: JSON schema file path to validate given config file
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend

    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    config_path = anyconfig.utils.ensure_expandusr(config_path)

    cparser = find_loader(config_path, forced_type)
    if cparser is None:
        return None

    if ac_schema is not None:
        kwargs["ac_schema"] = None  # Avoid infinit loop
        format_checker = kwargs.get("format_checker", None)
        LOGGER.info("Loading schema: %s", ac_schema)
        schema = load(ac_schema, forced_type=forced_type,
                      ignore_missing=ignore_missing, ac_template=ac_template,
                      ac_context=ac_context, **kwargs)

    LOGGER.info("Loading: %s", config_path)
    if ac_template:
        try:
            LOGGER.debug("Compiling: %s", config_path)
            config_content = anyconfig.template.render(config_path, ac_context)
            config = cparser.loads(config_content,
                                   ignore_missing=ignore_missing, **kwargs)
            if ac_schema is not None:
                if not _validate(config, schema, format_checker):
                    return None

            return config

        except Exception as exc:
            LOGGER.debug("Exc=%s", str(exc))
            LOGGER.warn("Failed to compile %s, fallback to no template "
                        "mode", config_path)

    config = cparser.load(config_path, ignore_missing=ignore_missing, **kwargs)

{'fraction_non_alphanumeric': 0.07954545454545454, 'fraction_numerical': 0.002403846153846154, 'mean_word_length': 3.0179982440737487, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3777324', 'n_tokens_mistral': 2638, 'n_tokens_neox': 2588, 'n_words': 790}