<commit_before>"""
Genducers in Python are pseudo-transducers that have identical semantics in
call to genduce (analogous to call to transduce). I.e., they can be composed
with a left-to-right processing chain in the first argument to genduce.

:TODO:
    Need to investigate speedup options from itertools, etc. These are mostly
    naive imperative implementations of generators, builtins from itertools
    when available (e.g. dropwhile, takewhile) are probably faster.
 
    Tests to determine equivalence with transducers (in tests/)

    Tests to determine input and output type/context capabilities.
"""
# Build this to reduce over the final generator, compose to get generator.
from functools import partial, reduce
from random import random

def compose(*fns):
    """Inverted from transducers due to generator semantics."""
    return reduce(lambda f,g: lambda x: g(f(x)), fns)


def taking(n):
    """Taking transducer."""
    def generator(coll):
        for i, item in enumerate(coll):
            if i >= n:
                raise StopIteration
            yield item
    return generator


def cat(coll):
    for item in coll:
        for subitem in item:
            yield subitem


def mapping(f):
    """Some functools can handle generator input correctly."""
    return partial(map, f)


def filtering(f):
    """Some functools can handle generator input correctly."""
    return partial(filter, f)


def remove(pred):
    """Remove genducer."""
    def generator(coll):
        for item in coll:
            if not pred(item):
                yield item
    return generator


def take_while(pred):
    """Keep taking until pred is false."""
    def generator(coll):
        for item in coll:
            if pred(item):
                yield item
            else:
                raise StopIteration
    return generator

def drop(n):
    """Drop until n"""
    def generator(coll):
        for i, item in enumerate(coll):
            if i >= n:
                yield item
    return generator

def drop_while(pred):
    def generator(coll):
        trigger = False
        for item in coll:
            if trigger:
                yield item
            elif not pred(item):
                trigger = True
                yield item
    return generator

def take_nth(n):
    def generator(coll):
        for i, item in enumerate(coll):
            if not i % n:
                yield item
    return generator

def replace(smap):
    def generator(coll):
        for item in coll:
            if item in smap:
                yield smap[item]
            else:
                yield item
    return generator

def keep(pred):
    def generator(coll):
        for item in coll:
            res = pred(item)
            if res is not None:
                yield res
    return generator

def keep_indexed(f):
    def generator(coll):
        for idx, item in enumerate(coll):
            res = f(idx, item)
            if res is not None:
                yield res
    return generator

def dedupe(coll):
    prev = next(coll)
    yield prev
    for item in coll:
        if item != prev:
            prev = item
            yield item

def partition_by(pred):
    def generator(coll):
        last = False
        temp = []
        for item in coll:
            if pred(item) == last:
                temp.append(item)
            else:
                yield (a for a in temp)
                temp = [item]
                last = pred(item)
    return generator

def partition_all(n):
    def generator(coll):
        temp = []
        for i, item in enumerate(coll, 1):
            temp.append(item)
            if not i % n:
                yield (a for a in temp)
                temp = []
        if temp:
            yield (a for a in temp)
    return generator

def random_sample(prob):
    def generator(coll):
        for item in coll:
            if random() < prob:
                yield item
    return generator

def mapcatting(f):
    return compose(mapping(f), cat)


def genduce(generator, reducer, start, coll):
    return reduce(reducer, generator((a for a in coll)), start)<commit_msg>catching cython up to latest changes.<commit_after>"""
Genducers in Python are pseudo-transducers that have identical semantics in
call to genduce (analogous to call to transduce). I.e., they can be composed
with a left-to-right processing chain in the first argument to genduce.

Note: cgenducers.pyx should only be generated by copying this file:

sh> cp transducers/genducers.py transducers/cgenducers.pyx

Do not change cgenducers.pyx directly at this point!

:TODO:
    Need to investigate speedup options from itertools, etc. These are mostly
    naive imperative implementations of generators, builtins from itertools
    when available (e.g. dropwhile, takewhile) are probably faster.
"""
# Build this to reduce over the final generator, compose to get generator.
from functools import partial, reduce
from random import random

def compose(*fns):
    """Inverted from transducers due to generator semantics."""
    return reduce(lambda f,g: lambda x: g(f(x)), fns)


def taking(n):
    """Taking transducer."""
    def generator(coll):
        for i, item in enumerate(coll):
            if i >= n:
                break
            yield item
    return generator


def cat(coll):
    for item in coll:
        for subitem in item:
            yield subitem


def mapping(f):
    """Some functools can handle generator input correctly."""
    return partial(map, f)


def filtering(f):
    """Some functools can handle generator input correctly."""
    return partial(filter, f)


def remove(pred):
    """Remove genducer."""
    def generator(coll):
        for item in coll:
            if not pred(item):
                yield item
    return generator


def take_while(pred):
    """Keep taking until pred is false."""
    def generator(coll):
        for item in coll:
            if pred(item):
                yield item
            else:
                break
    return generator

def drop(n):
    """Drop until n"""
    def generator(coll):
        for i, item in enumerate(coll):
            if i >= n:
                yield item
    return generator

def drop_while(pred):
    def generator(coll):
        trigger = False
        for item in coll:
            if trigger:
                yield item
            elif not pred(item):
                trigger = True
                yield item
    return generator

def take_nth(n):
    def generator(coll):
        for i, item in enumerate(coll):
            if not i % n:
                yield item
    return generator

def replace(smap):
    def generator(coll):
        for item in coll:
            if item in smap:
                yield smap[item]
            else:
                yield item
    return generator

def keep(pred):
    def generator(coll):
        for item in coll:
            res = pred(item)
            if res is not None:
                yield res
    return generator

def keep_indexed(f):
    def generator(coll):
        for idx, item in enumerate(coll):
            res = f(idx, item)
            if res is not None:
                yield res
    return generator

def dedupe(coll):
    prev = next(coll)
    yield prev
    for item in coll:
        if item != prev:
            prev = item
            yield item

def partition_by(pred):
    def generator(coll):
        last = False
        temp = []
        for item in coll:
            if pred(item) == last:
                temp.append(item)
            else:
                yield temp
                temp = [item]
                last = pred(item)
        if temp:
            yield temp
    return generator

def partition_all(n):
    def generator(coll):
        temp = []
        for i, item in enumerate(coll, 1):
            temp.append(item)
            if not i % n:
                yield temp
                temp = []
        if temp:
            yield temp
    return generator

def random_sample(prob):
    def generator(coll):
        for item in coll:
            if random() < prob:
                yield item
    return generator

def mapcatting(f):
    return compose(mapping(f), cat)


def transduce(generator, reducer, start, coll):
    return reduce(reducer, generator((a for a in coll)), start)
{'fraction_non_alphanumeric': 0.07869366112316221, 'fraction_numerical': 0.00024102193299590263, 'mean_word_length': 1.8304911323328785, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5294698', 'n_tokens_mistral': 2275, 'n_tokens_neox': 2170, 'n_words': 882}