<commit_before>from __builtin__ import all as _all, any as _any
from collections import Mapping, Set, Iterable, Iterator
from itertools import ifilter, imap, chain

from .funcs import complement
from .seqs import take


__all__ = ['empty', 'iteritems', 'join', 'merge',
           'walk', 'walk_keys', 'walk_values', 'select', 'select_keys', 'select_values',
           'is_distinct', 'all', 'any', 'none', 'one', 'some',
           'zipdict', 'flip', 'project']


### Generic ops
def empty(coll):
    return coll.__class__()

def iteritems(coll):
    return coll.iteritems() if hasattr(coll, 'iteritems') else coll


def join(colls):
    it = iter(colls)
    dest = next(it, None)
    if dest is None:
        raise TypeError('join needs at least one collection or string')
    cls = dest.__class__

    if isinstance(dest, basestring):
        return ''.join(colls)
    elif isinstance(dest, Mapping):
        return reduce(lambda a, b: cls(a, **b), colls)
    elif isinstance(dest, Set):
        return dest.union(*it)
    elif isinstance(dest, (Iterator, xrange)):
        return chain.from_iterable(colls)
    elif isinstance(dest, Iterable):
        return cls(chain.from_iterable(colls)) # could be reduce(concat, ...)
                                               # more effective for low count
    else:
        raise TypeError("Don't know how to join %s" % cls.__name__)

def merge(*colls):
    return join(colls)

# postponed
# def conj(coll, *xs):
#     return merge(coll, xs)


def walk(f, coll):
    """
    Walk coll transforming it's elements with f.
    Same as map, but preserves coll type.
    """
    return coll.__class__(imap(f, iteritems(coll)))

def walk_keys(f, coll):
    return walk(lambda (k, v): (f(k), v), coll)

def walk_values(f, coll):
    return walk(lambda (k, v): (k, f(v)), coll)

# TODO: prewalk, postwalk and friends

def select(f, coll):
    """Same as filter but preserves coll type."""
    return coll.__class__(ifilter(f, iteritems(coll)))

def select_keys(f, coll):
    return select(lambda (k, v): f(k), coll)

def select_values(f, coll):
    return select(lambda (k, v): f(v), coll)

### Content tests

def is_distinct(coll):
    return len(coll) == len(set(coll))


def all(pred, coll=None):
    if coll is None:
        return _all(pred)
    return _all(imap(pred, coll))

def any(pred, coll=None):
    if coll is None:
        return _any(pred)
    return _any(imap(pred, coll))

none = complement(any)

def one(pred, coll=None):
    if coll is None:
        return one(bool, pred)
    return len(take(2, ifilter(pred, coll))) == 1

# Not same as in clojure! returns value found not pred(value)
# NOTE: should I name it "find" when pred is here
def some(pred, coll=None):
    if coll is None:
        return some(None, pred)
    return next(ifilter(pred, coll), None)

# TODO: vector comparison tests - ascending, descending and such

# TODO: capabilities + type tests or skip?

def zipdict(keys, vals):
    return dict(zip(keys, vals))

def flip(mapping):
    return walk(lambda (k, v): (v, k), mapping)

def project(mapping, keys):
    return mapping.__class__((k, mapping[k]) for k in keys if k in mapping)
<commit_msg>Named predicate `pred` in `select_*()`<commit_after>from __builtin__ import all as _all, any as _any
from collections import Mapping, Set, Iterable, Iterator
from itertools import ifilter, imap, chain

from .funcs import complement
from .seqs import take


__all__ = ['empty', 'iteritems', 'join', 'merge',
           'walk', 'walk_keys', 'walk_values', 'select', 'select_keys', 'select_values',
           'is_distinct', 'all', 'any', 'none', 'one', 'some',
           'zipdict', 'flip', 'project']


### Generic ops
def empty(coll):
    return coll.__class__()

def iteritems(coll):
    return coll.iteritems() if hasattr(coll, 'iteritems') else coll


def join(colls):
    it = iter(colls)
    dest = next(it, None)
    if dest is None:
        raise TypeError('join needs at least one collection or string')
    cls = dest.__class__

    if isinstance(dest, basestring):
        return ''.join(colls)
    elif isinstance(dest, Mapping):
        return reduce(lambda a, b: cls(a, **b), colls)
    elif isinstance(dest, Set):
        return dest.union(*it)
    elif isinstance(dest, (Iterator, xrange)):
        return chain.from_iterable(colls)
    elif isinstance(dest, Iterable):
        return cls(chain.from_iterable(colls)) # could be reduce(concat, ...)
                                               # more effective for low count
    else:
        raise TypeError("Don't know how to join %s" % cls.__name__)

def merge(*colls):
    return join(colls)

# postponed
# def conj(coll, *xs):
#     return merge(coll, xs)


def walk(f, coll):
    """
    Walk coll transforming it's elements with f.
    Same as map, but preserves coll type.
    """
    return coll.__class__(imap(f, iteritems(coll)))

def walk_keys(f, coll):
    return walk(lambda (k, v): (f(k), v), coll)

def walk_values(f, coll):
    return walk(lambda (k, v): (k, f(v)), coll)

# TODO: prewalk, postwalk and friends

def select(pred, coll):
    """Same as filter but preserves coll type."""
    return coll.__class__(ifilter(pred, iteritems(coll)))

def select_keys(pred, coll):
    return select(lambda (k, v): pred(k), coll)

def select_values(pred, coll):
    return select(lambda (k, v): pred(v), coll)

### Content tests

def is_distinct(coll):
    return len(coll) == len(set(coll))


def all(pred, coll=None):
    if coll is None:
        return _all(pred)
    return _all(imap(pred, coll))

def any(pred, coll=None):
    if coll is None:
        return _any(pred)
    return _any(imap(pred, coll))

none = complement(any)

def one(pred, coll=None):
    if coll is None:
        return one(bool, pred)
    return len(take(2, ifilter(pred, coll))) == 1

# Not same as in clojure! returns value found not pred(value)
# NOTE: should I name it "find" when pred is here
def some(pred, coll=None):
    if coll is None:
        return some(None, pred)
    return next(ifilter(pred, coll), None)

# TODO: vector comparison tests - ascending, descending and such

# TODO: capabilities + type tests or skip?

def zipdict(keys, vals):
    return dict(zip(keys, vals))

def flip(mapping):
    return walk(lambda (k, v): (v, k), mapping)

def project(mapping, keys):
    return mapping.__class__((k, mapping[k]) for k in keys if k in mapping)

{'fraction_non_alphanumeric': 0.13261312040081416, 'fraction_numerical': 0.0006262721152340692, 'mean_word_length': 3.3016835016835016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5747122', 'n_tokens_mistral': 2078, 'n_tokens_neox': 2047, 'n_words': 698}