<commit_before>from __future__ import absolute_import, division, print_function
import numpy as np
from itertools import chain, cycle, izip
from .util_inject import inject
print, print_, printDBG, rrr, profile = inject(__name__, '[iter]')


def ensure_iterable(obj):
    if np.iterable(obj):
        return obj
    else:
        return [obj]


def ifilter_items(item_iter, flag_iter):
    filtered_items = (item for (item, flag) in
                      izip(item_iter, flag_iter)
                      if flag)
    return filtered_items


def ifilter_Nones(iter_):
    """ Removes any nones from the iterable """
    return (item for item in iter_ if item is not None)


def isiterable(obj):
    return np.iterable(obj) and not isinstance(obj, (str, unicode))


def iflatten(list_):
    """ flatten """
    flat_iter = chain.from_iterable(list_)  # very fast flatten
    return flat_iter


def iflatten_scalars(list_):
    [item for item in list_]


def ichunks(list_, size):
    'Yield successive n-sized chunks from list_.'
    for ix in xrange(0, len(list_), size):
        yield list_[ix: ix + size]


def interleave(args):
    arg_iters = map(iter, args)
    cycle_iter = cycle(arg_iters)
    for iter_ in cycle_iter:
        yield iter_.next()
<commit_msg>docstring on util_iter<commit_after>from __future__ import absolute_import, division, print_function
import numpy as np
from itertools import chain, cycle, izip
from .util_inject import inject
print, print_, printDBG, rrr, profile = inject(__name__, '[iter]')


def ensure_iterable(obj):
    if np.iterable(obj):
        return obj
    else:
        return [obj]


def ifilter_items(item_iter, flag_iter):
    filtered_items = (item for (item, flag) in
                      izip(item_iter, flag_iter)
                      if flag)
    return filtered_items


def ifilter_Nones(iter_):
    """ Removes any nones from the iterable """
    return (item for item in iter_ if item is not None)


def isiterable(obj):
    return np.iterable(obj) and not isinstance(obj, (str, unicode))


def iflatten(list_):
    """ flatten """
    flat_iter = chain.from_iterable(list_)  # very fast flatten
    return flat_iter


def iflatten_scalars(list_):
    [item for item in list_]


def ichunks(list_, chunksize):
    """ Yield successive n-sized chunks from list_."""
    for ix in xrange(0, len(list_), chunksize):
        yield list_[ix: ix + chunksize]


def interleave(args):
    arg_iters = map(iter, args)
    cycle_iter = cycle(arg_iters)
    for iter_ in cycle_iter:
        yield iter_.next()

{'fraction_non_alphanumeric': 0.09393346379647749, 'fraction_numerical': 0.0007827788649706458, 'mean_word_length': 3.1901639344262294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7276612', 'n_tokens_mistral': 837, 'n_tokens_neox': 808, 'n_words': 258}