<commit_before>from abc import ABCMeta, abstractmethod
import six
import collections

from picklable_itertools.version import __version__


@six.add_metaclass(ABCMeta)
class BaseItertool(six.Iterator):
    def __iter__(self):
        return self

    @abstractmethod
    def __next__(self):
        pass


def _iter(obj):
    """A custom replacement for iter(), dispatching a few custom picklable
    iterators for known types.
    """
    if six.PY2 and isinstance(obj, (list, tuple)):
        return ordered_sequence_iterator(obj)
    else:
        return iter(obj)


class ordered_sequence_iterator(BaseItertool):
    """A picklable replacement for list and tuple iterators."""
    def __init__(self, sequence):
        self._sequence = sequence
        self._position = 0

    def __next__(self):
        if self._position < len(self._sequence):
            value = self._sequence[self._position]
            self._position += 1
            return value
        else:
            raise StopIteration


class repeat(BaseItertool):
    """<commit_msg>Add file_iterator<commit_after>from abc import ABCMeta, abstractmethod
import collections
import io
import six

from picklable_itertools.version import __version__  # noqa


@six.add_metaclass(ABCMeta)
class BaseItertool(six.Iterator):
    def __iter__(self):
        return self

    @abstractmethod
    def __next__(self):
        pass


def _iter(obj):
    """A custom replacement for iter(), dispatching a few custom picklable
    iterators for known types.
    """
    if six.PY2:
        file_types = file,
    if six.PY3:
        file_types = io.IOBase,

    if six.PY2 and isinstance(obj, (list, tuple)):
        return ordered_sequence_iterator(obj)
    if isinstance(obj, file_types):
        return file_iterator(obj)
    else:
        return iter(obj)


class file_iterator(BaseItertool):
    """A picklable file iterator."""
    def __init__(self, f):
        self._f = f

    def __next__(self):
        line = self._f.readline()
        if not line:
            raise StopIteration
        return line

    def __getstate__(self):
        name, pos, mode = self._f.name, self._f.tell(), self._f.mode
        return name, pos, mode

    def __setstate__(self, state):
        name, pos, mode = state
        self._f = open(name, mode=mode)
        self._f.seek(pos)


class ordered_sequence_iterator(BaseItertool):
    """A picklable replacement for list and tuple iterators."""
    def __init__(self, sequence):
        self._sequence = sequence
        self._position = 0

    def __next__(self):
        if self._position < len(self._sequence):
            value = self._sequence[self._position]
            self._position += 1
            return value
        else:
            raise StopIteration


class repeat(BaseItertool):
    """
{'fraction_non_alphanumeric': 0.08980755523877405, 'fraction_numerical': 0.002851033499643621, 'mean_word_length': 2.5895140664961636, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6589849', 'n_tokens_mistral': 813, 'n_tokens_neox': 794, 'n_words': 230}