<commit_before>""" 
Reusable utility functions
"""

import numpy as np

from six.moves import range, zip
from functools import partial
from itertools import tee

def nwise(iterable, n):
    """
    Iterator that acts like a sliding window of size `n`; slides over
    some iterable `n` items at a time. If iterable has `m` elements, 
    this function will return an iterator over `m-n+1` tuples.

    Parameters
    ----------
    iterable : iterable
        An iterable object.
    
    n : int
        Window size.
    
    Returns
    -------
    iterator of tuples.
        Iterator of size `n` tuples 

    Notes
    -----
        
    First n iterators are created::

        iters = tee(iterable, n)

    Next, iterator i is advanced i times::

        for i, it in enumerate(iters):
            for _ in range(i):
                next(it, None)

    Then, the iterators are zipped back up again::

        return zip(*iters)

    Examples<commit_msg>utility<commit_after>""" 
Reusable utility functions
"""

import numpy as np

from six.moves import range, zip
from functools import partial
from itertools import tee

def nwise(iterable, n):
    """
    Iterator that acts like a sliding window of size `n`; slides over
    some iterable `n` items at a time. If iterable has `m` elements, 
    this function will return an iterator over `m-n+1` tuples.

    Parameters
    ----------
    iterable : iterable
        An iterable object.
    
    n : int
        Window size.
    
    Returns
    -------
    iterator of tuples.
        Iterator of size `n` tuples 

    Notes
    -----
        
    First `n` iterators are created::

        iters = tee(iterable, n)

    Next, iterator `i` is advanced `i` times::

        for i, it in enumerate(iters):
            for _ in range(i):
                next(it, None)

    Then, the iterators are zipped back up again::

        return zip(*iters)

    Examples
{'fraction_non_alphanumeric': 0.09280500521376434, 'fraction_numerical': 0.0010427528675703858, 'mean_word_length': 1.9297709923664121, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '191138', 'n_tokens_mistral': 582, 'n_tokens_neox': 549, 'n_words': 225}