<commit_before>from .base import BaseItertool
from .iter_dispatch import _iter


class _grouper(BaseItertool):
    def __init__(self, iterator, value, keyfunc):
        self._iterator = iterator
        self._value = value
        self._keyfunc = keyfunc
        self._key = self._keyfunc(self._value)
        self._initialized = False
        self._stream_ended = False
        self._done = False

    def __next__(self):
        if not self._initialized:
            self._initialized = True
            return self._value
        else:
            if self._done:
                raise StopIteration
            try:
                value = next(self._iterator)
            except StopIteration:
                self._stream_ended = True
                self._done = True
                raise
            if self._keyfunc(value) != self._key:
                self._terminal_value = value
                self._done = True
                raise StopIteration
            return value


class groupby(BaseItertool):
    """groupby(iterable[, keyfunc]) -> create an iterator which returns
    (key, sub-iterator) grouped by each value of key(value).
    """
    def __init__(self, iterable, key=None):
        self._key = key
        self._iterable = _iter(iterable)
        self._current_key = self._initial_key = object()

    def _keyfunc(self, value):
        if self._key is None:
            return value
        else:
            return self._key(value)

    def __next__(self):
        if not hasattr(self, '_current_grouper'):
            value = next(self._iterable)
            self._current_grouper = _grouper(self._iterable, value,
                                             self._keyfunc)
            return self._keyfunc(value), self._current_grouper
        else:
            while True:
                try:
                    next(self._current_grouper)
                except StopIteration:
                    break
            if self._current_grouper._stream_ended:
                raise StopIteration
            value = self._current_grouper._terminal_value
            self._current_grouper = _grouper(self._iterable, value,
                                             self._keyfunc)
            return self._keyfunc(value), self._current_grouper<commit_msg>Fix pickling on Python 2.x.<commit_after>from .base import BaseItertool
from .iter_dispatch import _iter


class _grouper(BaseItertool):
    def __init__(self, value, groupby):
        self._value = value
        self._groupby = groupby
        self._key = self._groupby._keyfunc(self._value)
        self._initialized = False
        self._stream_ended = False
        self._done = False

    def __next__(self):
        if not self._initialized:
            self._initialized = True
            return self._value
        else:
            if self._done:
                raise StopIteration
            try:
                value = next(self._groupby._iterator)
            except StopIteration:
                self._stream_ended = True
                self._done = True
                raise
            if self._groupby._keyfunc(value) != self._key:
                self._terminal_value = value
                self._done = True
                raise StopIteration
            return value


class groupby(BaseItertool):
    """groupby(iterable[, keyfunc]) -> create an iterator which returns
    (key, sub-iterator) grouped by each value of key(value).
    """
    def __init__(self, iterable, key=None):
        self._key = key
        self._iterator = _iter(iterable)
        self._current_key = self._initial_key = object()

    def _keyfunc(self, value):
        if self._key is None:
            return value
        else:
            return self._key(value)

    def __next__(self):
        if not hasattr(self, '_current_grouper'):
            value = next(self._iterator)
            self._current_grouper = _grouper(value, self)
            return self._keyfunc(value), self._current_grouper
        else:
            while True:
                try:
                    next(self._current_grouper)
                except StopIteration:
                    break
            if self._current_grouper._stream_ended:
                raise StopIteration
            value = self._current_grouper._terminal_value
            self._current_grouper = _grouper(value, self)
            return self._keyfunc(value), self._current_grouper
{'fraction_non_alphanumeric': 0.07447528774542993, 'fraction_numerical': 0.00022568269013766644, 'mean_word_length': 1.6459701492537313, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2207393', 'n_tokens_mistral': 1177, 'n_tokens_neox': 1137, 'n_words': 278}