<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals


def chunk_iter(iterable, n):
    """Yields an iterator in chunks

    For example you can do

    for a, b in chunk_iter([1, 2, 3, 4, 5, 6], 2):
        print '{0} {1}'.format(a, b)

    # Prints
    # 1 2
    # 3 4
    # 5 6

    Args:
        iterable - Some iterable
        n - Chunk size (must be greater than 0)
    """
    assert n > 0
    iterable = tuple(iterable)

    chunk = iterable[:n]
    iterable = iterable[n:]
    while chunk:
        yield chunk
        chunk = iterable[:n]
        iterable = iterable[n:]<commit_msg>Improve performance of chunk_iter<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

import itertools


def chunk_iter(iterable, n):
    """Yields an iterator in chunks

    For example you can do

    for a, b in chunk_iter([1, 2, 3, 4, 5, 6], 2):
        print('{0} {1}'.format(a, b))

    # Prints
    # 1 2
    # 3 4
    # 5 6

    Args:
        iterable - Some iterable
        n - Chunk size (must be greater than 0)
    """
    assert n > 0
    iterable = iter(iterable)

    chunk = tuple(itertools.islice(iterable, n))
    while chunk:
        yield chunk
        chunk = tuple(itertools.islice(iterable, n))
{'fraction_non_alphanumeric': 0.10061919504643962, 'fraction_numerical': 0.02631578947368421, 'mean_word_length': 2.332474226804124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595030', 'n_tokens_mistral': 469, 'n_tokens_neox': 427, 'n_words': 145}