<commit_before>import itertools
import collections
import operator

class Relation(object):
    def __init__(self, seq):
        self._iter = seq

    def param(self, name):
        ''' extracts param "name" from a dict

        >>> x = Relation([{'a': 1}, {'a': 2}])
        >>> list(x.param('a'))
        [1, 2]
        '''
        return itertools.imap(lambda o: o.get(name), self._iter)

    def params(self, *names):
        ''' returns tuples of projected names

        >>> x = Relation([{'a': 1, 'b': 2}]).params('a', 'b')
        >>> list(x)
        [(1, 2)]
        '''
        return itertools.imap(operator.itemgetter(*names), self._iter)

    def compute(self, paramDict):
        ''' computes additional parameters to the object, according paramDict

        >>> x = Relation([{'a': 1}]).compute({'c': lambda o: o['a'] + 1})
        >>> list(x)
        [{'a': 1, 'c': 2}]
        '''
        def setter(o):
            o.update(dict((name, func(o)) for name, func in paramDict.items()))
            return o
        return self.map(setter)

    def project(self, *names):
        ''' return subset of attributes

        >>> x = Relation([{'a': 1, 'b': 2, 'c': 3}]).project('a', 'b')
        >>> list(x)
        [{'a': 1, 'b': 2}]
        '''
        return self.map(lambda o: dict((k, o.get(k)) for k in names))

    def map(self, func):
        return self.__class__(itertools.imap(func, self._iter))

    def filter(self, func):
        ''' return only attributes accepted by function func

        >>> x = Relation([{'a': 1}, {'a': 2}, {'a': 3}]).filter(lambda o: o['a'] < 3)
        >>> list(x.param('a'))
        [1, 2]
        '''
        return self.__class__(itertools.ifilter(func, self._iter))

    def __iter__(self):
        for x in self._iter:
            yield x

    def rename(self, **kwargs):
        ''' rename some attributes according kwargs mapping

        >>> x = Relation([{'a': 1, 'b': 2}]).rename(a='c')
        >>> list(x)
        [{'c': 1, 'b': 2}]
        '''
        func = lambda o: dict((kwargs.get(k, k), o[k]) for k in o)
        return self.map(func)

    def fix(self):
        ''' materialize data so it is possible to iterate it twice

        >>> it = iter([{'a': 1, 'b': 2}])
        >>> rel = Relation(it).fix()
        >>> list(rel)
        [{'a': 1, 'b': 2}]
        >>> list(rel)
        [{'a': 1, 'b': 2}]
        '''
        return self.__class__(list(self._iter))

    def copy(self):
        return self.map(dict)

    def count(self):
        ''' returns number of elements

        >>> Relation([{'a': 1}, {'a': 2}]).count()
        2
        '''
        return len(self._iter)

    def sortBy(self, keyFunc):
        ''' sorts items

        >>> x = Relation([{'a': 3}, {'a': 1}, {'a': 2}]).sortBy(lambda o: o['a'])
        >>> list(x.param('a'))
        [1, 2, 3]
        '''
        array = list(self._iter)
        array.sort(key=keyFunc)
        return self.__class__(array)

    def sortByNames(self, *names):
        ''' sorts items

        >>> x = Relation([{'a': 3}, {'a': 1}, {'a': 2}]).sortByNames('a')
        >>> list(x.param('a'))
        [1, 2, 3]
        '''
        return self.sortBy(keyFunc=operator.itemgetter(*names))

    @staticmethod
    def _checkNonDecreasing(it, keyFunc):
        ascending = False
        descending = False

        it = iter(it)
        prev = it.next()
        yield prev
        prevValue = keyFunc(prev)
        while True:
            current = it.next()
            currentValue = keyFunc(current)
            if prevValue > currentValue:
                if ascending:
                    raise RuntimeError('the relation is not sorted')
                descending = True
            elif prevValue < currentValue:
                if descending:
                    raise RuntimeError('the relation is not sorted')
                ascending = True
            yield current
            prevValue = currentValue

    def groupBy(self, keyFunc, isSorted=False):
        ''' sorts and groups items return iterables
        '''
        if isSorted:
            it = self._checkNonDecreasing(self._iter, keyFunc)
        else:
            it = list(self._iter)
            it.sort(key=keyFunc)
        return self.__class__((k, list(g)) for k, g in itertools.groupby(it, keyFunc))

    def groupByNames(self, *names, **kwargs):
        ''' sorts and groups items

        Check that unsorted relation will throw an error
        >>> x = Relation([{'a': 2}, {'a': 1}, {'a': 2}]).groupByNames('a', isSorted=True)
        >>> list(x)
        Traceback (most recent call last):
        RuntimeError: the relation is not sorted

        Check that sorted relation, can be returned without any problem (ascending)
        >>> x = Relation([{'a': 1}, {'a': 1}, {'a': 2}]).groupByNames('a', isSorted=True)
        >>> list(x)
        [(1, [{'a': 1}, {'a': 1}]), (2, [{'a': 2}])]

        Check that sorted relation, can be returned without any problem (descending)
        >>> x = Relation([{'a': 2}, {'a': 1}, {'a': 1}]).groupByNames('a', isSorted=True)
        >>> list(x)
        [(2, [{'a': 2}]), (1, [{'a': 1}, {'a': 1}])]

        Check the grouping with sorting
        >>> x = Relation([{'a': 2}, {'a': 1}, {'a': 2}]).groupByNames('a')
        >>> list(x)
        [(1, [{'a': 1}]), (2, [{'a': 2}, {'a': 2}])]
        '''
        return self.groupBy(keyFunc=operator.itemgetter(*names), isSorted=kwargs.get('isSorted', False))

    def mapping(self, keyFunc):
        ''' returns mapping of key to value
        '''
        mapping = {}
        for v in self._iter:
            k = keyFunc(v)
            if k in mapping:
                raise RuntimeError('mapping key is not unique')
            mapping[k] = v

    def mappingByNames(self, *names):
        return self.mapping(keyFunc=operator.itemgetter(*names))

    def takeWhile(self, func):
        ''' take arguments while func holds and forget the rest
        '''
        return self.__class__(itertools.takewhile(func, self._iter))

    def dropWhile(self, func):
        ''' skip arguments until func holds and returns the rest
        '''
        return self.__class__(itertools.dropwhile(func, self._iter))

    def skip(self, n):
        ''' skip first n arguments and returns the rest
        '''<commit_msg>F mapping: missing return, add tests<commit_after>import itertools
import collections
import operator

class Relation(object):
    def __init__(self, seq):
        self._iter = seq

    def param(self, name):
        ''' extracts param "name" from a dict

        >>> x = Relation([{'a': 1}, {'a': 2}])
        >>> list(x.param('a'))
        [1, 2]
        '''
        return itertools.imap(lambda o: o.get(name), self._iter)

    def params(self, *names):
        ''' returns tuples of projected names

        >>> x = Relation([{'a': 1, 'b': 2}]).params('a', 'b')
        >>> list(x)
        [(1, 2)]
        '''
        return itertools.imap(operator.itemgetter(*names), self._iter)

    def compute(self, paramDict):
        ''' computes additional parameters to the object, according paramDict

        >>> x = Relation([{'a': 1}]).compute({'c': lambda o: o['a'] + 1})
        >>> list(x)
        [{'a': 1, 'c': 2}]
        '''
        def setter(o):
            o.update(dict((name, func(o)) for name, func in paramDict.items()))
            return o
        return self.map(setter)

    def project(self, *names):
        ''' return subset of attributes

        >>> x = Relation([{'a': 1, 'b': 2, 'c': 3}]).project('a', 'b')
        >>> list(x)
        [{'a': 1, 'b': 2}]
        '''
        return self.map(lambda o: dict((k, o.get(k)) for k in names))

    def map(self, func):
        return self.__class__(itertools.imap(func, self._iter))

    def filter(self, func):
        ''' return only attributes accepted by function func

        >>> x = Relation([{'a': 1}, {'a': 2}, {'a': 3}]).filter(lambda o: o['a'] < 3)
        >>> list(x.param('a'))
        [1, 2]
        '''
        return self.__class__(itertools.ifilter(func, self._iter))

    def __iter__(self):
        for x in self._iter:
            yield x

    def rename(self, **kwargs):
        ''' rename some attributes according kwargs mapping

        >>> x = Relation([{'a': 1, 'b': 2}]).rename(a='c')
        >>> list(x)
        [{'c': 1, 'b': 2}]
        '''
        func = lambda o: dict((kwargs.get(k, k), o[k]) for k in o)
        return self.map(func)

    def fix(self):
        ''' materialize data so it is possible to iterate it twice

        >>> it = iter([{'a': 1, 'b': 2}])
        >>> rel = Relation(it).fix()
        >>> list(rel)
        [{'a': 1, 'b': 2}]
        >>> list(rel)
        [{'a': 1, 'b': 2}]
        '''
        return self.__class__(list(self._iter))

    def copy(self):
        return self.map(dict)

    def count(self):
        ''' returns number of elements

        >>> Relation([{'a': 1}, {'a': 2}]).count()
        2
        '''
        return len(self._iter)

    def sortBy(self, keyFunc):
        ''' sorts items

        >>> x = Relation([{'a': 3}, {'a': 1}, {'a': 2}]).sortBy(lambda o: o['a'])
        >>> list(x.param('a'))
        [1, 2, 3]
        '''
        array = list(self._iter)
        array.sort(key=keyFunc)
        return self.__class__(array)

    def sortByNames(self, *names):
        ''' sorts items

        >>> x = Relation([{'a': 3}, {'a': 1}, {'a': 2}]).sortByNames('a')
        >>> list(x.param('a'))
        [1, 2, 3]
        '''
        return self.sortBy(keyFunc=operator.itemgetter(*names))

    @staticmethod
    def _checkNonDecreasing(it, keyFunc):
        ascending = False
        descending = False

        it = iter(it)
        prev = it.next()
        yield prev
        prevValue = keyFunc(prev)
        while True:
            current = it.next()
            currentValue = keyFunc(current)
            if prevValue > currentValue:
                if ascending:
                    raise RuntimeError('the relation is not sorted')
                descending = True
            elif prevValue < currentValue:
                if descending:
                    raise RuntimeError('the relation is not sorted')
                ascending = True
            yield current
            prevValue = currentValue

    def groupBy(self, keyFunc, isSorted=False):
        ''' sorts and groups items return iterables
        '''
        if isSorted:
            it = self._checkNonDecreasing(self._iter, keyFunc)
        else:
            it = list(self._iter)
            it.sort(key=keyFunc)
        return self.__class__((k, list(g)) for k, g in itertools.groupby(it, keyFunc))

    def groupByNames(self, *names, **kwargs):
        ''' sorts and groups items

        Check that unsorted relation will throw an error
        >>> x = Relation([{'a': 2}, {'a': 1}, {'a': 2}]).groupByNames('a', isSorted=True)
        >>> list(x)
        Traceback (most recent call last):
        RuntimeError: the relation is not sorted

        Check that sorted relation, can be returned without any problem (ascending)
        >>> x = Relation([{'a': 1}, {'a': 1}, {'a': 2}]).groupByNames('a', isSorted=True)
        >>> list(x)
        [(1, [{'a': 1}, {'a': 1}]), (2, [{'a': 2}])]

        Check that sorted relation, can be returned without any problem (descending)
        >>> x = Relation([{'a': 2}, {'a': 1}, {'a': 1}]).groupByNames('a', isSorted=True)
        >>> list(x)
        [(2, [{'a': 2}]), (1, [{'a': 1}, {'a': 1}])]

        Check the grouping with sorting
        >>> x = Relation([{'a': 2}, {'a': 1}, {'a': 2}]).groupByNames('a')
        >>> list(x)
        [(1, [{'a': 1}]), (2, [{'a': 2}, {'a': 2}])]
        '''
        return self.groupBy(keyFunc=operator.itemgetter(*names), isSorted=kwargs.get('isSorted', False))

    def mapping(self, keyFunc):
        ''' returns mapping of key to value

        Test mapping according a custom key
        >>> f = lambda o: o['a'] * 3
        >>> Relation([{'a': 1}, {'a': 2}, {'a': 3}]).mapping(f)
        {9: {'a': 3}, 3: {'a': 1}, 6: {'a': 2}}

        Test duplicit key
        >>> Relation([{'a': 1}, {'a': 2}, {'a': 2}]).mapping(f)
        Traceback (most recent call last):
        RuntimeError: mapping key is not unique
        '''
        mapping = {}
        for v in self._iter:
            k = keyFunc(v)
            if k in mapping:
                raise RuntimeError('mapping key is not unique')
            mapping[k] = v
        return mapping

    def mappingByNames(self, *names):
        ''' mapping version with key equal to names
        >>> Relation([{'a': 1}, {'a': 2}, {'a': 3}]).mappingByNames('a')
        {1: {'a': 1}, 2: {'a': 2}, 3: {'a': 3}}
        '''
        return self.mapping(keyFunc=operator.itemgetter(*names))

    def takeWhile(self, func):
        ''' take arguments while func holds and forget the rest
        '''
        return self.__class__(itertools.takewhile(func, self._iter))

    def dropWhile(self, func):
        ''' skip arguments until func holds and returns the rest
        '''
        return self.__class__(itertools.dropwhile(func, self._iter))

    def skip(self, n):
        ''' skip first n arguments and returns the rest
        '''
{'fraction_non_alphanumeric': 0.17742547219904423, 'fraction_numerical': 0.013047106121520139, 'mean_word_length': 2.0511455681555195, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 191, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5950002', 'n_tokens_mistral': 4259, 'n_tokens_neox': 4016, 'n_words': 1177}