<commit_before>import collections
import itertools
import six

def flatten(sequence_of_sequences):
    return itertools.chain.from_iterable(sequence_of_sequences)


def pairwise(sequence):
    sequence = iter(sequence)
    try:
        while True:
            yield next(sequence), next(sequence)
    except StopIteration:
        pass


def ordered_map(sequence):
    return collections.OrderedDict(sorted(pairwise(sequence)))


class annotate(object):
    __slots__ = ('_o',)
    __merky__ = True

    def __init__(self, wrapped):
        self._o = wrapped

    def __getattr__(self, attr):
        return getattr(self._o, attr)

    def __iter__(self):
        return iter(self._o)


def annotate_values(dictlike):
    return dict((k, annotate(v)) for k, v in six.iteritems(dictlike))
<commit_msg>Docs for annotation utils<commit_after>import collections
import itertools
import six

def flatten(sequence_of_sequences):
    return itertools.chain.from_iterable(sequence_of_sequences)


def pairwise(sequence):
    sequence = iter(sequence)
    try:
        while True:
            yield next(sequence), next(sequence)
    except StopIteration:
        pass


def ordered_map(sequence):
    return collections.OrderedDict(sorted(pairwise(sequence)))


class annotate(object):
    """
    Tells merky to tokenize a given object.

    Wrap any arbitrary object with this; the wrapper delegates
    to the wrapped object, but provides a `__merky__` attribute of
    value `True`, indicating to transformers that the object should
    be "tokenized".
    """
    __slots__ = ('_o',)
    __merky__ = True

    def __init__(self, wrapped):
        self._o = wrapped

    def __getattr__(self, attr):
        return getattr(self._o, attr)

    def __iter__(self):
        return iter(self._o)


def annotate_values(dictlike):
    """
    Returns a dictionary based on `dictlike` with each value wrapped by `annotate`.

    The dictionary itself is not wrapped by `annotate`, only the values.
    """
    return dict((k, annotate(v)) for k, v in six.iteritems(dictlike))

{'fraction_non_alphanumeric': 0.0910411622276029, 'fraction_numerical': 0.0, 'mean_word_length': 3.3771186440677967, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3043956', 'n_tokens_mistral': 631, 'n_tokens_neox': 604, 'n_words': 179}