<commit_before>from persistent import Persistent
from itertools import islice, izip, imap, count
from cachetools import LRUCache
from zerodb.storage import prefetch


class Sliceable(object):
    def __init__(self, f, cache_size=1000, length=None):
        """
        Makes a sliceable, cached list-like interface to an iterator
        :param callable f: Function which inits the iterator
        """
        self.f = f
        self.cache = LRUCache(cache_size)
        self.stop = 0
        self.length = length
        self.iterator = iter(f())

    def __iter__(self):
        self.iterator = iter(self.f())
        for i in count():
            y = self.__getitem__(i)
            yield y
            # We get StopIteration error once we're done

    def dictify(self):
        for obj in self.__iter__():
            if hasattr(obj, "_p_activate"):
                obj._p_activate()
            yield {k: v for k, v in obj.__dict__.iteritems() if not k.startswith("_")}

    def __len__(self):
        if self.length is None:
            if hasattr(self.iterator, "__len__"):
                return len(self.iterator)
            else:
                return len(list(self.__iter__()))
        else:<commit_msg>Don't create an iterator once again<commit_after>from persistent import Persistent
from itertools import islice, izip, imap, count
from cachetools import LRUCache
from zerodb.storage import prefetch


class Sliceable(object):
    def __init__(self, f, cache_size=1000, length=None):
        """
        Makes a sliceable, cached list-like interface to an iterator
        :param callable f: Function which inits the iterator
        """
        self.f = f
        self.cache = LRUCache(cache_size)
        self.stop = 0
        self.length = length
        self.iterator = iter(f())

    def __iter__(self):
        for i in count():
            y = self.__getitem__(i)
            yield y
            # We get StopIteration error once we're done

    def dictify(self):
        for obj in self.__iter__():
            if hasattr(obj, "_p_activate"):
                obj._p_activate()
            yield {k: v for k, v in obj.__dict__.iteritems() if not k.startswith("_")}

    def __len__(self):
        if self.length is None:
            if hasattr(self.iterator, "__len__"):
                return len(self.iterator)
            else:
                return len(list(self.__iter__()))
        else:
{'fraction_non_alphanumeric': 0.09342703596527491, 'fraction_numerical': 0.00413393964448119, 'mean_word_length': 2.0440251572327046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7445168', 'n_tokens_mistral': 708, 'n_tokens_neox': 682, 'n_words': 218}