<commit_before>import copy
import threading

from contextlib import contextmanager
from functools import wraps
from string import Template

import blinker

from bunch import Bunch, bunchify

from loader import Loader
from monitor import SourceMonitor


def synchronized(lock):
    def wrap(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            _lock = getattr(args[0], lock) if isinstance(lock, basestring) \
                                           else lock
            with _lock:
                return func(*args, **kwargs)
        return wrapper
    return wrap


class FrozenError(Exception):
    pass


class Frozen(Bunch):
    def __setattr__(self, k, v):
        raise FrozenError()

    def __setitem__(self, k, v):
        raise FrozenError()


class Config(Bunch):
    def __init__(self, *args, **kwargs):
        super(Config, self).__init__(*args, **kwargs)
        self.__dict__['_frozen'] = False
        for item in [k for k in self.keys() if not k.startswith('_')]:
            if isinstance(self[item], dict):
                self[item] = bunchify(self[item])

    def __setattr__(self, k, v):
        if self._frozen:
            raise FrozenError()
        super(Config, self).__setattr__(k, v)

    def __setitem__(self, k, v):
        if self._frozen:
            raise FrozenError()
        super(Config, self).__setitem__(k, v)

    @classmethod
    def _traverse(cls, direction, config, callback, keys=None):
        def _handle_dict(keys, val, parent=None):
            if direction == 'down':
                callback(keys, val, parent, not isinstance(val, dict))
                if len(keys) > 0:
                    val = parent[keys[-1]]
                cls._traverse(direction, val, callback, keys)
            elif direction == 'up':
                cls._traverse(direction, val, callback, keys)
                callback(keys, val, parent, not isinstance(val, dict))
            else:
                raise ValueError('unknown direction %s' % (direction))
        if keys is None:
            keys = []
            _handle_dict(keys, config)
        elif isinstance(config, (dict, Frozen)):
            for key in sorted(config.keys()):
                keys.append(key)
                _handle_dict(keys, config[key], config)
                keys.pop()

    @classmethod
    def _walk(cls, config, callback, keys=None):
        try:
            cls._traverse('down', config, callback, keys)
        except StopIteration:
            pass

    @classmethod
    def _moon_walk(cls, config, callback, keys=None):
        try:
            cls._traverse('up', config, callback, keys)
        except StopIteration:
            pass

    def _merge(self, other):
        """ Merge another config into this one

        """
        def _merge_node(keys, val, parent, isleaf):
            if not isleaf:
                return
            node = self
            for i in xrange(len(keys) - 1):
                if keys[i] not in node or \
                   not isinstance(node[keys[i]], dict):
                    for key in keys[i:-1]:
                        node[key] = Bunch()
                        node = node[key]
                    break
                node = node[keys[i]]
            node[keys[-1]] = val

        with self._handle():
            Config._walk(other, _merge_node)

    def _do_subs(self, sub_key):
        def __do_subs(node, subs):
            if isinstance(node, dict):
                for k in node.keys():
                    if node == self and k == sub_key:
                        continue
                    node[k] = __do_subs(node[k], subs)
            elif isinstance(node, list):
                for i in xrange(len(node)):
                    node[i] = __do_subs(node[i], subs)
            elif isinstance(node, basestring):
                node = Template(node).substitute(**subs)
            return node

        if sub_key in self.keys():
            with self._handle():
                __do_subs(self, self[sub_key])

    def _freeze(self):
        """ Make Config immutable

        NOTE: embedded lists are still mutable

        """
        if self._frozen:
            return
        def _freeze_node(keys, val, parent, isleaf):
            if val is not self and isinstance(val, dict):
                parent[keys[-1]] = Frozen(val)
        Config._moon_walk(self, _freeze_node)
        self._frozen = True

    def _thaw(self):
        """ Make Config mutable

        """
        if not self._frozen:
            return
        def _thaw_node(keys, val, parent, isleaf):<commit_msg>remove substitution block before traversing the config<commit_after>import copy
import threading

from contextlib import contextmanager
from functools import wraps
from string import Template

import blinker

from bunch import Bunch, bunchify

from loader import Loader
from monitor import SourceMonitor


def synchronized(lock):
    def wrap(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            _lock = getattr(args[0], lock) if isinstance(lock, basestring) \
                                           else lock
            with _lock:
                return func(*args, **kwargs)
        return wrapper
    return wrap


class FrozenError(Exception):
    pass


class Frozen(Bunch):
    def __setattr__(self, k, v):
        raise FrozenError()

    def __setitem__(self, k, v):
        raise FrozenError()


class Config(Bunch):
    def __init__(self, *args, **kwargs):
        super(Config, self).__init__(*args, **kwargs)
        self.__dict__['_frozen'] = False
        for item in [k for k in self.keys() if not k.startswith('_')]:
            if isinstance(self[item], dict):
                self[item] = bunchify(self[item])

    def __setattr__(self, k, v):
        if self._frozen:
            raise FrozenError()
        super(Config, self).__setattr__(k, v)

    def __setitem__(self, k, v):
        if self._frozen:
            raise FrozenError()
        super(Config, self).__setitem__(k, v)

    @classmethod
    def _traverse(cls, direction, config, callback, keys=None):
        def _handle_dict(keys, val, parent=None):
            if direction == 'down':
                callback(keys, val, parent, not isinstance(val, dict))
                if len(keys) > 0:
                    val = parent[keys[-1]]
                cls._traverse(direction, val, callback, keys)
            elif direction == 'up':
                cls._traverse(direction, val, callback, keys)
                callback(keys, val, parent, not isinstance(val, dict))
            else:
                raise ValueError('unknown direction %s' % (direction))
        if keys is None:
            keys = []
            _handle_dict(keys, config)
        elif isinstance(config, (dict, Frozen)):
            for key in sorted(config.keys()):
                keys.append(key)
                _handle_dict(keys, config[key], config)
                keys.pop()

    @classmethod
    def _walk(cls, config, callback, keys=None):
        try:
            cls._traverse('down', config, callback, keys)
        except StopIteration:
            pass

    @classmethod
    def _moon_walk(cls, config, callback, keys=None):
        try:
            cls._traverse('up', config, callback, keys)
        except StopIteration:
            pass

    def _merge(self, other):
        """ Merge another config into this one

        """
        def _merge_node(keys, val, parent, isleaf):
            if not isleaf:
                return
            node = self
            for i in xrange(len(keys) - 1):
                if keys[i] not in node or \
                   not isinstance(node[keys[i]], dict):
                    for key in keys[i:-1]:
                        node[key] = Bunch()
                        node = node[key]
                    break
                node = node[keys[i]]
            node[keys[-1]] = val

        with self._handle():
            Config._walk(other, _merge_node)

    def _do_subs(self, sub_key):
        def __do_subs(node, subs):
            if isinstance(node, dict):
                for k in node.keys():
                    if node == self and k == sub_key:
                        continue
                    node[k] = __do_subs(node[k], subs)
            elif isinstance(node, list):
                for i in xrange(len(node)):
                    node[i] = __do_subs(node[i], subs)
            elif isinstance(node, basestring):
                node = Template(node).substitute(**subs)
            return node

        if sub_key in self.keys():
            with self._handle():
                subs = self.pop(sub_key)
                try:
                    __do_subs(self, subs)
                finally:
                    self[sub_key] = subs

    def _freeze(self):
        """ Make Config immutable

        NOTE: embedded lists are still mutable

        """
        if self._frozen:
            return
        def _freeze_node(keys, val, parent, isleaf):
            if val is not self and isinstance(val, dict):
                parent[keys[-1]] = Frozen(val)
        Config._moon_walk(self, _freeze_node)
        self._frozen = True

    def _thaw(self):
        """ Make Config mutable

        """
        if not self._frozen:
            return
        def _thaw_node(keys, val, parent, isleaf):
{'fraction_non_alphanumeric': 0.10158424320274032, 'fraction_numerical': 0.00149860843502462, 'mean_word_length': 1.6663812785388128, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1736311', 'n_tokens_mistral': 2649, 'n_tokens_neox': 2529, 'n_words': 739}