<commit_before>import copy
import os
from os.path import join
from types import DictType, BooleanType, StringTypes, ListType, TupleType

from .vendor.etcaetera.config import Config as EtcConfig
from .vendor.etcaetera.adapter import File, Defaults, Overrides, Adapter

from .exceptions import AmbiguousEnvVar, UncastableEnvVar
from .util import debug


def noop(s):
    """
    No-op 'formatter' for etcaetera adapters.

    For when we do not want auto upper/lower casing (currently, always).
    """
    return s


class NestedEnv(Adapter):
    """
    Custom etcaetera adapter for handling env vars (more flexibly than Env).
    """
    def __init__(self, config, prefix=None):
        """
        Initialize this adapter with a handle to a live Config object.

        :param config:
            An already-loaded Config object which can be introspected for its
            keys.

        :param str prefix:
            A prefix string used when seeking env vars; see `.Config`.
        """
        super(NestedEnv, self).__init__()
        self._config = config
        self._prefix = prefix

    def load(self, formatter=None):
        # NOTE: This accepts a formatter argument because that's the API.
        # However we don't use it (or the centrally defined one) because we
        # have specific requirements for how keys are treated in this adapter.
        # Meh!

        # Obtain allowed env var -> existing value map
        env_vars = self._crawl(key_path=[], env_vars={})
        # Check for actual env var (honoring prefix) and try to set
        for env_var, key_path in env_vars.iteritems():
            real_var = (self._prefix or "") + env_var
            if real_var in os.environ:
                self._path_set(key_path, os.environ[real_var])

    def _crawl(self, key_path, env_vars):
        """
        Examine config at location ``key_path`` & return potential env vars.

        Uses ``env_vars`` dict to determine if a conflict exists, and raises an
        exception if so. This dict is of the following form::

            {
                'EXPECTED_ENV_VAR_HERE': ['actual', 'nested', 'key_path'],
                ...
            }

        Returns another dictionary of new keypairs as per above.
        """
        new_vars = {}
        obj = self._path_get(key_path)
        # Sub-dict -> recurse
        if hasattr(obj, 'keys') and hasattr(obj, '__getitem__'):
            for key in obj.keys():
                merged_vars = dict(env_vars, **new_vars)
                merged_path = key_path + [key]
                crawled = self._crawl(merged_path, merged_vars)
                # Handle conflicts
                for key in crawled:
                    if key in new_vars:
                        err = "Found >1 source for {0}"
                        raise AmbiguousEnvVar(err.format(key))
                # Merge and continue
                new_vars.update(crawled)
        # Other -> is leaf, no recursion
        else:
            new_vars[self._to_env_var(key_path)] = key_path
        return new_vars

    def _to_env_var(self, key_path):
        return '_'.join(key_path).upper()

    def _path_get(self, key_path):
        obj = self._config
        for key in key_path:
            obj = obj[key]
        return obj

    def _path_set(self, key_path, value):
        obj = self._config
        for key in key_path[:-1]:
            obj = obj[key]
        old = obj[key_path[-1]]
        obj[key_path[-1]] = self._cast(old, value)

    def _cast(self, old, new_):
        if isinstance(old, BooleanType):
            return new_ not in ('0', '')
        elif isinstance(old, StringTypes):
            return new_
        elif old is None:
            return new_
        elif isinstance(old, (ListType, TupleType)):
            err = "Can't adapt an environment string into a {0}!"
            err = err.format(type(old))
            raise UncastableEnvVar(err)
        else:
            return old.__class__(new_)<commit_msg>Modify self.data instead of mutating handle on config like an idiot<commit_after>import copy
import os
from os.path import join
from types import DictType, BooleanType, StringTypes, ListType, TupleType

from .vendor.etcaetera.config import Config as EtcConfig
from .vendor.etcaetera.adapter import File, Defaults, Overrides, Adapter

from .exceptions import AmbiguousEnvVar, UncastableEnvVar
from .util import debug


def noop(s):
    """
    No-op 'formatter' for etcaetera adapters.

    For when we do not want auto upper/lower casing (currently, always).
    """
    return s


class NestedEnv(Adapter):
    """
    Custom etcaetera adapter for handling env vars (more flexibly than Env).
    """
    def __init__(self, config, prefix=None):
        """
        Initialize this adapter with a handle to a live Config object.

        :param config:
            An already-loaded Config object which can be introspected for its
            keys.

        :param str prefix:
            A prefix string used when seeking env vars; see `.Config`.
        """
        super(NestedEnv, self).__init__()
        self._config = config
        self._prefix = prefix
        self.data = {}

    def load(self, formatter=None):
        # NOTE: This accepts a formatter argument because that's the API.
        # However we don't use it (or the centrally defined one) because we
        # have specific requirements for how keys are treated in this adapter.
        # Meh!

        # Obtain allowed env var -> existing value map
        env_vars = self._crawl(key_path=[], env_vars={})
        # Check for actual env var (honoring prefix) and try to set
        for env_var, key_path in env_vars.iteritems():
            real_var = (self._prefix or "") + env_var
            if real_var in os.environ:
                self._path_set(key_path, os.environ[real_var])

    def _crawl(self, key_path, env_vars):
        """
        Examine config at location ``key_path`` & return potential env vars.

        Uses ``env_vars`` dict to determine if a conflict exists, and raises an
        exception if so. This dict is of the following form::

            {
                'EXPECTED_ENV_VAR_HERE': ['actual', 'nested', 'key_path'],
                ...
            }

        Returns another dictionary of new keypairs as per above.
        """
        new_vars = {}
        obj = self._path_get(key_path)
        # Sub-dict -> recurse
        if hasattr(obj, 'keys') and hasattr(obj, '__getitem__'):
            for key in obj.keys():
                merged_vars = dict(env_vars, **new_vars)
                merged_path = key_path + [key]
                crawled = self._crawl(merged_path, merged_vars)
                # Handle conflicts
                for key in crawled:
                    if key in new_vars:
                        err = "Found >1 source for {0}"
                        raise AmbiguousEnvVar(err.format(key))
                # Merge and continue
                new_vars.update(crawled)
        # Other -> is leaf, no recursion
        else:
            new_vars[self._to_env_var(key_path)] = key_path
        return new_vars

    def _to_env_var(self, key_path):
        return '_'.join(key_path).upper()

    def _path_get(self, key_path):
        # Gets are from self._config because that's what determines valid env
        # vars and/or values for typecasting.
        obj = self._config
        for key in key_path:
            obj = obj[key]
        return obj

    def _path_set(self, key_path, value):
        # Sets are to self.data since that's what we are presenting to the
        # outer config object and debugging.
        obj = self.data
        for key in key_path[:-1]:
            if key not in obj:
                obj[key] = {}
            obj = obj[key]
        old = self._path_get(key_path)
        new_ = self._cast(old, value)
        obj[key_path[-1]] = new_

    def _cast(self, old, new_):
        if isinstance(old, BooleanType):
            return new_ not in ('0', '')
        elif isinstance(old, StringTypes):
            return new_
        elif old is None:
            return new_
        elif isinstance(old, (ListType, TupleType)):
            err = "Can't adapt an environment string into a {0}!"
            err = err.format(type(old))
            raise UncastableEnvVar(err)
        else:
            return old.__class__(new_)
{'fraction_non_alphanumeric': 0.08715925394548063, 'fraction_numerical': 0.0015542802486848398, 'mean_word_length': 1.9705255681818181, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5583746', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2313, 'n_words': 796}