<commit_before>import itertools

## Copyright (c) Cognitect, Inc.
## All rights reserved.

def mapcat(f, seq):
    for x in seq:
        for y in f(x):
            yield y

def pairs(i):
    for x in range(0, len(i), 2):
        yield (i[x], i[x+1])

def cycle(x):
    d = tuple(x)
    while True:
        for x in d:
            yield x

def take(n, s):
    c = 0
    for x in s:
        yield x
        c += 1
        if c >= n:
            break<commit_msg>All iter/seq operations should minimize overhead with itertools functions<commit_after>import itertools

## Copyright (c) Cognitect, Inc.
## All rights reserved.

def mapcat(f, i):
    return itertools.chain.from_iterable(itertools.imap(f, i))

def pairs(i):
    return zip(*[iter(i)] * 2)

cycle = itertools.cycle

def take(n, i):
    return itertools.islice(s, 0, n)

{'fraction_non_alphanumeric': 0.12545235223160434, 'fraction_numerical': 0.008443908323281062, 'mean_word_length': 2.3333333333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6300295', 'n_tokens_mistral': 300, 'n_tokens_neox': 301, 'n_words': 98}