<commit_before>from collections import Mapping, MutableMapping, MutableSequence, Sequence

from django.utils.module_loading import import_string

from six import string_types

from .base import Base
from .checker import Checker
from .settings import DottedAccessDict, Settings
from .types import LocalSetting


class Loader(Base):

    def __init__(self, file_name, section=None, registry=None, strategy_type=None):
        super(Loader, self).__init__(file_name, section, registry, strategy_type)

    def load_and_check(self, base_settings, prompt=None):
        """Load settings and check them.

        Loads the settings from ``base_settings``, then checks them.

        Returns:
            (merged settings, True) on success
            (None, False) on failure

        """
        checker = Checker(self.file_name, self.section, self.registry, self.strategy_type, prompt)
        settings = self.load(base_settings)
        if checker.check(settings):
            return settings, True
        return None, False

    def load(self, base_settings):
        """Merge local settings from file with ``base_settings``.

        Returns a new settings dict containing the base settings and the
        loaded settings. Includes:

            - base settings
            - settings from extended file(s), if any
            - settings from file

        """
        is_valid_key = lambda k: k.isupper() and not k.startswith('_')

        # Base settings, including `LocalSetting`s, loaded from the
        # Django settings module.
        valid_keys = (k for k in base_settings if is_valid_key(k))
        base_settings = DottedAccessDict((k, base_settings[k]) for k in valid_keys)

        # Settings read from the settings file; values are unprocessed.
        settings_from_file, _ = self.strategy.read_file(self.file_name, self.section)

        # The fully resolved settings.
        settings = Settings(base_settings)

        settings_names = []
        settings_not_decoded = set()

        for name, value in settings_from_file.items():
            for prefix in ('PREPEND.', 'APPEND.', 'SWAP.'):
                if name.startswith(prefix):
                    name = name[len(prefix):]
                    name = '{prefix}({name})'.format_map(locals())
                    break

            settings_names.append(name)

            # Attempt to decode raw values. Errors in decoding at this
            # stage are ignored.
            try:
                value = self.strategy.decode_value(value)
            except ValueError:
                settings_not_decoded.add(name)

            settings.set_dotted(name, value)

            # See if this setting corresponds to a `LocalSetting`. If
            # so, note that the `LocalSetting` has a value by putting it
            # in the registry. This also makes it easy to retrieve the
            # `LocalSetting` later so its value can be set.
            current_value = base_settings.get_dotted(name, None)
            if isinstance(current_value, LocalSetting):
                self.registry[current_value] = name

        # Interpolate values of settings read from file. When a setting
        # that couldn't be decoded previously is encountered, its post-
        # interpolation value will be decoded.
        for name in settings_names:
            value = settings.get_dotted(name)
            value, _ = self._interpolate_values(value, settings)
            if name in settings_not_decoded:
                value = self.strategy.decode_value(value)
            settings.set_dotted(name, value)

        # Interpolate base settings.
        self._interpolate_values(settings, settings)

        self._interpolate_keys(settings, settings)

        self._prepend_extras(settings, settings.pop('PREPEND', None))
        self._append_extras(settings, settings.pop('APPEND', None))
        self._swap_list_items(settings, settings.pop('SWAP', None))
        self._import_from_string(settings, settings.pop('IMPORT_FROM_STRING', None))
        self._delete_settings(settings, settings.pop('DELETE', None))

        for local_setting, name in self.registry.items():
            local_setting.value = settings.get_dotted(name)

        return settings

    # Post-processing

    def _interpolate_values(self, obj, settings):
        all_interpolated = []
        interpolated = []
        while interpolated is not None:
            all_interpolated.extend(interpolated)
            obj, interpolated = self._interpolate_values_inner(obj, settings)
        return obj, all_interpolated

    def _interpolate_values_inner(self, obj, settings, _interpolated=None):
        if _interpolated is None:
            _interpolated = []

        if isinstance(obj, string_types):
            new_value, changed = self._inject(obj, settings)
            if changed:
                _interpolated.append((obj, new_value))
                obj = new_value
        elif isinstance(obj, MutableMapping):
            for k, v in obj.items():
                v, _interpolated = self._interpolate_values_inner(v, settings, _interpolated)
                obj[k] = v
        elif isinstance(obj, Mapping):
            items = []
            for k, v in obj.items():
                v, _interpolated = self._interpolate_values_inner(v, settings, _interpolated)
                items.append((k, v))
            obj = obj.__class__(items)
        elif isinstance(obj, MutableSequence):
            for i, v in enumerate(obj):
                v, _interpolated = self._interpolate_values_inner(v, settings, _interpolated)
                obj[i] = v
        elif isinstance(obj, Sequence):
            items = []
            for v in obj:
                v, _interpolated = self._interpolate_values_inner(v, settings, _interpolated)
                items.append(v)
            obj = obj.__class__(items)

        return obj, _interpolated or None

    def _interpolate_keys(self, obj, settings):
        if isinstance(obj, Mapping):
            replacements = {}
            for k, v in obj.items():
                if isinstance(k, string_types):
                    new_k, changed = self._inject(k, settings)
                    if changed:
                        replacements[k] = new_k
                self._interpolate_keys(v, settings)
            for k, new_k in replacements.items():
                obj[new_k] = obj[k]
                del obj[k]
        elif isinstance(obj, Sequence) and not isinstance(obj, string_types):
            for item in obj:
                self._interpolate_keys(item, settings)<commit_msg>Simplify internals of interpolation in Loader<commit_after>from collections import Mapping, MutableMapping, MutableSequence, Sequence

from django.utils.module_loading import import_string

from six import string_types

from .base import Base
from .checker import Checker
from .settings import DottedAccessDict, Settings
from .types import LocalSetting


class Loader(Base):

    def __init__(self, file_name, section=None, registry=None, strategy_type=None):
        super(Loader, self).__init__(file_name, section, registry, strategy_type)

    def load_and_check(self, base_settings, prompt=None):
        """Load settings and check them.

        Loads the settings from ``base_settings``, then checks them.

        Returns:
            (merged settings, True) on success
            (None, False) on failure

        """
        checker = Checker(self.file_name, self.section, self.registry, self.strategy_type, prompt)
        settings = self.load(base_settings)
        if checker.check(settings):
            return settings, True
        return None, False

    def load(self, base_settings):
        """Merge local settings from file with ``base_settings``.

        Returns a new settings dict containing the base settings and the
        loaded settings. Includes:

            - base settings
            - settings from extended file(s), if any
            - settings from file

        """
        is_valid_key = lambda k: k.isupper() and not k.startswith('_')

        # Base settings, including `LocalSetting`s, loaded from the
        # Django settings module.
        valid_keys = (k for k in base_settings if is_valid_key(k))
        base_settings = DottedAccessDict((k, base_settings[k]) for k in valid_keys)

        # Settings read from the settings file; values are unprocessed.
        settings_from_file, _ = self.strategy.read_file(self.file_name, self.section)

        # The fully resolved settings.
        settings = Settings(base_settings)

        settings_names = []
        settings_not_decoded = set()

        for name, value in settings_from_file.items():
            for prefix in ('PREPEND.', 'APPEND.', 'SWAP.'):
                if name.startswith(prefix):
                    name = name[len(prefix):]
                    name = '{prefix}({name})'.format_map(locals())
                    break

            settings_names.append(name)

            # Attempt to decode raw values. Errors in decoding at this
            # stage are ignored.
            try:
                value = self.strategy.decode_value(value)
            except ValueError:
                settings_not_decoded.add(name)

            settings.set_dotted(name, value)

            # See if this setting corresponds to a `LocalSetting`. If
            # so, note that the `LocalSetting` has a value by putting it
            # in the registry. This also makes it easy to retrieve the
            # `LocalSetting` later so its value can be set.
            current_value = base_settings.get_dotted(name, None)
            if isinstance(current_value, LocalSetting):
                self.registry[current_value] = name

        # Interpolate values of settings read from file. When a setting
        # that couldn't be decoded previously is encountered, its post-
        # interpolation value will be decoded.
        for name in settings_names:
            value = settings.get_dotted(name)
            value, _ = self._interpolate_values(value, settings)
            if name in settings_not_decoded:
                value = self.strategy.decode_value(value)
            settings.set_dotted(name, value)

        # Interpolate base settings.
        self._interpolate_values(settings, settings)

        self._interpolate_keys(settings, settings)

        self._prepend_extras(settings, settings.pop('PREPEND', None))
        self._append_extras(settings, settings.pop('APPEND', None))
        self._swap_list_items(settings, settings.pop('SWAP', None))
        self._import_from_string(settings, settings.pop('IMPORT_FROM_STRING', None))
        self._delete_settings(settings, settings.pop('DELETE', None))

        for local_setting, name in self.registry.items():
            local_setting.value = settings.get_dotted(name)

        return settings

    # Post-processing

    def _interpolate_values(self, obj, settings):
        def inject(value):
            new_value, changed = self._inject(value, settings)
            if changed:
                interpolated.append((value, new_value))
            return new_value

        while True:
            interpolated = []
            obj = self._traverse_object(obj, action=inject)
            if not interpolated:
                break

        return obj

    def _traverse_object(self, obj, action):
        if isinstance(obj, string_types):
            obj = action(obj)
        elif isinstance(obj, MutableMapping):
            for k, v in obj.items():
                v = self._traverse_object(v, action)
                obj[k] = v
        elif isinstance(obj, Mapping):
            items = []
            for k, v in obj.items():
                v = self._traverse_object(v, action)
                items.append((k, v))
            obj = obj.__class__(items)
        elif isinstance(obj, MutableSequence):
            for i, v in enumerate(obj):
                v = self._traverse_object(v, action)
                obj[i] = v
        elif isinstance(obj, Sequence):
            items = []
            for v in obj:
                v = self._traverse_object(v, action)
                items.append(v)
            obj = obj.__class__(items)
        return obj

    def _interpolate_keys(self, obj, settings):
        if isinstance(obj, Mapping):
            replacements = {}
            for k, v in obj.items():
                if isinstance(k, string_types):
                    new_k, changed = self._inject(k, settings)
                    if changed:
                        replacements[k] = new_k
                self._interpolate_keys(v, settings)
            for k, new_k in replacements.items():
                obj[new_k] = obj[k]
                del obj[k]
        elif isinstance(obj, Sequence) and not isinstance(obj, string_types):
            for item in obj:
                self._interpolate_keys(item, settings)
{'fraction_non_alphanumeric': 0.08416802546781582, 'fraction_numerical': 0.0, 'mean_word_length': 2.1193993703075806, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4705411', 'n_tokens_mistral': 3487, 'n_tokens_neox': 3331, 'n_words': 978}