<commit_before>"""
.. drv.py

Dice random variables; these are discrete (integer-valued) finite random
variables.

This module provides a simple (quite thin) wrapper around the extensive
scipy.stats module.
"""

## SciPy's random variable framework
import scipy.stats as ss

## Math
import numpy as np
inf = np.inf

## Data containers
import collections as col

## Python basics
import functools as fn
import itertools as it

## For arithmetic
import operator as op


##################################
## ----- Helper Functions ----- ##
##################################

def unzip(zipped):
    """ Return a :term:`generator` reverses the work of zip/izip.

    :param zipped: the iterable to unzip.
    :type zipped: :term:`iterable` of :term:`iterables <iterable>`.
    :rtype: :term:`generator`

    Examples:

    >>> list(unzip(zip(xrange(3), xrange(2, 5))))
    [(0, 1, 2), (2, 3, 4)]
    >>> list(unzip(izip(xrange(3), xrange(2, 5))))
    [(0, 1, 2), (2, 3, 4)]

    .. note:: The returned elements of the generator are always tuples. This is
        a result of how :func:`zip` works.
    """
    return it.izip(*zipped)


##################################
## ----- Operator Classes ----- ##
##################################

class Operator(object):
    """ An :class:`Operator` is a class which acts efficiently on a pool of
    random variables, returning a random variable. """
    def __init__(self, operator, unpack=False):
        if not unpack:
            self.operator = operator
        else:
            self.operator = lambda x: operator(*x)

    def __call__(self, pool, name):
        return self._operate(pool, name)

    def _operate(self, pool, name):
        """ This method is intended to be overwritten by subclasses, for more
        efficient calculations. If not overwritten, a naive calculation is
        being executed, which may take exponential time. """
        d = col.defaultdict(float)

        for xp in it.product(*(drv.values for drv in pool.drvs)):
            xs, ps = unzip(xp)
            val = self.operator(xs)
            ival = int(val)
            if not val == ival:
                raise ValueError("Operator must return integer values.")
            d[ival] += reduce(op.mul, ps)

        formatter = dict(("_{i}".format(i=i), drv) for i, drv in
                         enumerate(pool.drvs))
        _name = name.format(**formatter)

        _xs = d.keys()
        _ps = d.values()
        return DiscreteRandomVariable(_name, xs=_xs, ps=_ps)


class ReduceOperator(Operator):
    """ This is a class of operators which may be reduced to simple memoryless
    binary operators. """
    def __init__(self, operator, identity=None, unpack=False):
        super(ReduceOperator, self).__init__(operator, unpack=unpack)
        self.identity = identity

    def _operate(self, pool, name):
        if not pool:
            if self.identity:
                return constant(self.identity)
            raise ValueError

        drvs = pool.drvs
        res = drvs[0]
        for rv in drvs[1:]:
            _pool = RandomVariablePool(res, rv)
            res = super(ReduceOperator, self)._operate(_pool, name)

        return res


class IndexedOperator(Operator):
    """ This is a class of operators which work only on a pre-defined set of
    indices of random variables from the pool. """
    def __init__(self, operator, indices, unpack=False):
        super(IndexedOperator, self).__init__(operator, unpack=unpack)
        self.indices = indices

    def _operator(self, pool):
        drvs = pool.drvs
        _drvs = [drvs[i] for i in self.indices]
        _pool = drv.RandomVariablePool(*_drvs)
        return super(IndexedOperator, self)._operate(_pool, name)<commit_msg>Added a TODO note about binary reduce.<commit_after>"""
.. drv.py

Dice random variables; these are discrete (integer-valued) finite random
variables.

This module provides a simple (quite thin) wrapper around the extensive
scipy.stats module.
"""

## SciPy's random variable framework
import scipy.stats as ss

## Math
import numpy as np
inf = np.inf

## Data containers
import collections as col

## Python basics
import functools as fn
import itertools as it

## For arithmetic
import operator as op


##################################
## ----- Helper Functions ----- ##
##################################

def unzip(zipped):
    """ Return a :term:`generator` reverses the work of zip/izip.

    :param zipped: the iterable to unzip.
    :type zipped: :term:`iterable` of :term:`iterables <iterable>`.
    :rtype: :term:`generator`

    Examples:

    >>> list(unzip(zip(xrange(3), xrange(2, 5))))
    [(0, 1, 2), (2, 3, 4)]
    >>> list(unzip(izip(xrange(3), xrange(2, 5))))
    [(0, 1, 2), (2, 3, 4)]

    .. note:: The returned elements of the generator are always tuples. This is
        a result of how :func:`zip` works.
    """
    return it.izip(*zipped)


##################################
## ----- Operator Classes ----- ##
##################################

class Operator(object):
    """ An :class:`Operator` is a class which acts efficiently on a pool of
    random variables, returning a random variable. """
    def __init__(self, operator, unpack=False):
        if not unpack:
            self.operator = operator
        else:
            self.operator = lambda x: operator(*x)

    def __call__(self, pool, name):
        return self._operate(pool, name)

    def _operate(self, pool, name):
        """ This method is intended to be overwritten by subclasses, for more
        efficient calculations. If not overwritten, a naive calculation is
        being executed, which may take exponential time. """
        d = col.defaultdict(float)

        for xp in it.product(*(drv.values for drv in pool.drvs)):
            xs, ps = unzip(xp)
            val = self.operator(xs)
            ival = int(val)
            if not val == ival:
                raise ValueError("Operator must return integer values.")
            d[ival] += reduce(op.mul, ps)

        formatter = dict(("_{i}".format(i=i), drv) for i, drv in
                         enumerate(pool.drvs))
        _name = name.format(**formatter)

        _xs = d.keys()
        _ps = d.values()
        return DiscreteRandomVariable(_name, xs=_xs, ps=_ps)


class ReduceOperator(Operator):
    """ This is a class of operators which may be reduced to simple memoryless
    binary operators. """
    def __init__(self, operator, identity=None, unpack=False):
        super(ReduceOperator, self).__init__(operator, unpack=unpack)
        self.identity = identity

    def _operate(self, pool, name):
        if not pool:
            if self.identity:
                return constant(self.identity)
            raise ValueError

        ## This is the naive implementation, but binary implementation won't be
        ## too difficult, and may be more efficient
        drvs = pool.drvs
        res = drvs[0]
        for rv in drvs[1:]:
            _pool = RandomVariablePool(res, rv)
            res = super(ReduceOperator, self)._operate(_pool, name)

        return res


class IndexedOperator(Operator):
    """ This is a class of operators which work only on a pre-defined set of
    indices of random variables from the pool. """
    def __init__(self, operator, indices, unpack=False):
        super(IndexedOperator, self).__init__(operator, unpack=unpack)
        self.indices = indices

    def _operator(self, pool):
        drvs = pool.drvs
        _drvs = [drvs[i] for i in self.indices]
        _pool = drv.RandomVariablePool(*_drvs)
        return super(IndexedOperator, self)._operate(_pool, name)
{'fraction_non_alphanumeric': 0.14497312885043911, 'fraction_numerical': 0.005243151133831433, 'mean_word_length': 2.680656054027979, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4525010', 'n_tokens_mistral': 2239, 'n_tokens_neox': 2141, 'n_words': 734}