<commit_before># -*- coding: utf-8 -*-

from itertools import imap, ifilter, islice, chain

import web
from sqlalchemy.sql import not_
from netaddr import IPSet, IPNetwork, iter_iprange

from nailgun.db import orm
from nailgun.task import errors
from nailgun.settings import settings
from nailgun.api.models import Node, IPAddr, Cluster
from nailgun.api.models import Network, NetworkGroup


def chunked_range(iterable, chunksize=1024):
    idx = 0
    while True:
        s = islice(iterable, idx, idx + chunksize)
        yield chain([s.next()], s)<commit_msg>Fixed broken chunked generator<commit_after># -*- coding: utf-8 -*-

from itertools import imap, ifilter, islice, chain, tee

import web
from sqlalchemy.sql import not_
from netaddr import IPSet, IPNetwork, iter_iprange

from nailgun.db import orm
from nailgun.task import errors
from nailgun.settings import settings
from nailgun.api.models import Node, IPAddr, Cluster
from nailgun.api.models import Network, NetworkGroup


def chunked_range(iterable, chunksize=1024):
    idx = 0
    while True:
        iterable, iterable2 = tee(iterable)
        s = islice(iterable2, idx, idx + chunksize)
        yield chain([s.next()], s)
{'fraction_non_alphanumeric': 0.08648194794290512, 'fraction_numerical': 0.011754827875734676, 'mean_word_length': 4.16017316017316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6325432', 'n_tokens_mistral': 384, 'n_tokens_neox': 370, 'n_words': 126}