<commit_before>"""Infrastructure for implementing transducers."""

from abc import ABCMeta, abstractmethod


class Reduced:
    """A sentinel 'box' used to return the final value of a reduction.

    Wrapping a value in a Reduced instance is idempotent, so
    Reduced(Reduced(item)) is equivalent to Reduced(item)
    """

    def __new__(cls, value):
        if isinstance(value, cls):
            return value
        obj = super().__new__(cls)
        obj._value = value
        return obj

    @property
    def value(self):
        return self._value


class Reducer(object, metaclass=ABCMeta):

    @abstractmethod
    def initial(self):
        raise NotImplementedError

    @abstractmethod
    def step(self, result, item):
        raise NotImplementedError

    def complete(self, result):
        return result

    def __call__(self, result, item):
        """Reducing objects are callable so they can be used like functions."""<commit_msg>Adds a __repr__ to Reduced.<commit_after>"""Infrastructure for implementing transducers."""

from abc import ABCMeta, abstractmethod


class Reduced:
    """A sentinel 'box' used to return the final value of a reduction.

    Wrapping a value in a Reduced instance is idempotent, so
    Reduced(Reduced(item)) is equivalent to Reduced(item)
    """

    def __new__(cls, value):
        if isinstance(value, cls):
            return value
        obj = super().__new__(cls)
        obj._value = value
        return obj

    @property
    def value(self):
        return self._value

    def __repr__(self):
        return '{}({!r})'.format(self.__class__.__name__, self._value)


class Reducer(object, metaclass=ABCMeta):

    @abstractmethod
    def initial(self):
        raise NotImplementedError

    @abstractmethod
    def step(self, result, item):
        raise NotImplementedError

    def complete(self, result):
        return result

    def __call__(self, result, item):
        """Reducing objects are callable so they can be used like functions."""
{'fraction_non_alphanumeric': 0.08928571428571429, 'fraction_numerical': 0.0, 'mean_word_length': 2.849236641221374, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2858512', 'n_tokens_mistral': 573, 'n_tokens_neox': 548, 'n_words': 192}