<commit_before>"""Core functionality of the lusmu library"""

# pylint: disable=W0212
#         Allow access to protected members of client classes
# pylint: disable=W0142
#         Allow * and ** magic

from collections import defaultdict
import itertools
import logging
import sys


LOG = logging.getLogger('lusmu.base')


if sys.version_info[0] == 2:
    def items(dictionary):
        """Return a set-like object, a view on key/value pairs of a dict"""
        return dictionary.viewitems()

    def values(dictionary):
        """Return an object providing a view on the values of a dictionary"""
        return dictionary.viewvalues()

    def get_func_name(function, default=None):
        """Return the name of the function, falling back to a default"""
        return getattr(function, 'func_name', default)
else:
    def items(dictionary):
        """Return a set-like object, a view on key/value pairs of a dict"""
        return dictionary.items()

    def values(dictionary):
        """Return an object providing a view on the values of a dictionary"""
        return dictionary.values()

    def get_func_name(function, default=None):
        """Return the name of the function, falling back to a default"""
        return getattr(function, '__name__', default)


_TRIGGERED_CACHE = {}


class DIRTY(object):
    """Class definition for the dirty node special value"""

    def __str__(self):
        return 'DIRTY'

    def __repr__(self):
        return '<lusmu.base.DIRTY>'
DIRTY = DIRTY()


class BaseNode(object):
    """Base class for Inputs and Nodes"""

    _name_counters = defaultdict(int)

    def __init__(self, name=None, value=DIRTY):
        self.name = name or self._generate_name()
        self._value = value
        self._dependents = set()

    def _connect(self, dependent):
        """Set the given Node as a dependent of this Node or Input

        Immediately paints the new dependent Node dirty if this Node has
        already been evaluated or if a value has already been set for this
        Input.

        Connecting Nodes always invalidates the triggered Nodes cache.

        """
        if dependent not in self._dependents:
            self._dependents.add(dependent)
            if self._value is not DIRTY:
                dependent._set_value(DIRTY, make_cache=False)
            _TRIGGERED_CACHE.clear()

    def _disconnect(self, dependent):
        """Remove given Node from the set of dependents of this Node or Input

        Immediately paints the new dependent Node dirty if this Node has
        previously been evaluated or if a value has previously been set for
        this Input.

        Disconnecting Nodes always invalidates the triggered nodes cache.

        """
        if dependent in self._dependents:
            self._dependents.remove(dependent)
            if self._value is not DIRTY:
                dependent._set_value(DIRTY, make_cache=False)
            _TRIGGERED_CACHE.clear()

    def _set_value(self, value, make_cache=True):
        """Set a new value for this Node or Input

        If this caused the value to change, paints dependent Nodes dirty and
        returns the set of those dependent Nodes which are marked "triggered"
        and should be re-evaluated.

        When called by ``set_value`` from external code, the ``make_cache``
        argument must be ``True`` so the return value is cached.  Internal
        calls set ``make_cache=False`` so memory isn't wasted for caching the
        triggered dependents of intermediate Nodes.

        This private method can be used as a debugging tool to set values of
        non-input Nodes.

        """
        if value == self._value:
            return set()
        self._value = value
        self._set_dependents_dirty()
        return self._get_triggered_dependents(make_cache=make_cache)

    def get_value(self):
        """Return the value of the object"""
        raise NotImplementedError('The get_value() method must be defined '<commit_msg>Made node value equality comparison overridable.<commit_after>"""Core functionality of the lusmu library"""

# pylint: disable=W0212
#         Allow access to protected members of client classes
# pylint: disable=W0142
#         Allow * and ** magic

from collections import defaultdict
import itertools
import logging
import sys


LOG = logging.getLogger('lusmu.base')


if sys.version_info[0] == 2:
    def items(dictionary):
        """Return a set-like object, a view on key/value pairs of a dict"""
        return dictionary.viewitems()

    def values(dictionary):
        """Return an object providing a view on the values of a dictionary"""
        return dictionary.viewvalues()

    def get_func_name(function, default=None):
        """Return the name of the function, falling back to a default"""
        return getattr(function, 'func_name', default)
else:
    def items(dictionary):
        """Return a set-like object, a view on key/value pairs of a dict"""
        return dictionary.items()

    def values(dictionary):
        """Return an object providing a view on the values of a dictionary"""
        return dictionary.values()

    def get_func_name(function, default=None):
        """Return the name of the function, falling back to a default"""
        return getattr(function, '__name__', default)


_TRIGGERED_CACHE = {}


class DIRTY(object):
    """Class definition for the dirty node special value"""

    def __str__(self):
        return 'DIRTY'

    def __repr__(self):
        return '<lusmu.base.DIRTY>'
DIRTY = DIRTY()


class BaseNode(object):
    """Base class for Inputs and Nodes"""

    _name_counters = defaultdict(int)

    def __init__(self, name=None, value=DIRTY):
        self.name = name or self._generate_name()
        self._value = value
        self._dependents = set()

    def _connect(self, dependent):
        """Set the given Node as a dependent of this Node or Input

        Immediately paints the new dependent Node dirty if this Node has
        already been evaluated or if a value has already been set for this
        Input.

        Connecting Nodes always invalidates the triggered Nodes cache.

        """
        if dependent not in self._dependents:
            self._dependents.add(dependent)
            if self._value is not DIRTY:
                dependent._set_value(DIRTY, make_cache=False)
            _TRIGGERED_CACHE.clear()

    def _disconnect(self, dependent):
        """Remove given Node from the set of dependents of this Node or Input

        Immediately paints the new dependent Node dirty if this Node has
        previously been evaluated or if a value has previously been set for
        this Input.

        Disconnecting Nodes always invalidates the triggered nodes cache.

        """
        if dependent in self._dependents:
            self._dependents.remove(dependent)
            if self._value is not DIRTY:
                dependent._set_value(DIRTY, make_cache=False)
            _TRIGGERED_CACHE.clear()

    def _set_value(self, value, make_cache=True):
        """Set a new value for this Node or Input

        If this caused the value to change, paints dependent Nodes dirty and
        returns the set of those dependent Nodes which are marked "triggered"
        and should be re-evaluated.

        When called by ``set_value`` from external code, the ``make_cache``
        argument must be ``True`` so the return value is cached.  Internal
        calls set ``make_cache=False`` so memory isn't wasted for caching the
        triggered dependents of intermediate Nodes.

        This private method can be used as a debugging tool to set values of
        non-input Nodes.

        """
        # test if neither, one of or both the old and the new value are DIRTY
        dirty_count = len([v for v in value, self._value if v is DIRTY])
        if dirty_count == 2:
            # both DIRTY, no need to touch anything
            return set()
        if dirty_count == 0 and self._value_eq(value):
            # both non-DIRTY but equal, no need to touch anything
            return set()
        # either one is DIRTY, or values aren't equal, update the value and
        # paint the dependent Nodes dirty
        self._value = value
        self._set_dependents_dirty()
        return self._get_triggered_dependents(make_cache=make_cache)

    def _value_eq(self, other_value):
        return self._value == other_value

    def get_value(self):
        """Return the value of the object"""
        raise NotImplementedError('The get_value() method must be defined '
{'fraction_non_alphanumeric': 0.07573109635325644, 'fraction_numerical': 0.002563206338110218, 'mean_word_length': 2.614315789473684, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1400915', 'n_tokens_mistral': 2305, 'n_tokens_neox': 2211, 'n_words': 917}