<commit_before>"""
Utilities related to testing.
"""

from __future__ import division

from boltons.iterutils import pairwise

import numpy as np

from hypothesis import assume
from hypothesis.extra.numpy import arrays
from hypothesis.strategies import composite, floats, integers, lists, tuples

from .. import Distribution

__all__ = ['distributions',
           'distribution_structures',
           'markov_chains',
          ]

@composite
def distributions(draw, alphabets=(2, 2, 2), nondegenerate=False):
    """
    Generate distributions for use with hypothesis.

    Parameters
    ----------
    draw : function
        A sampling function passed in by hypothesis.
    alphabets : int, tuple of ints, tuple of pairs of ints
        If an int, it is the length of the chain and each variable is assumed to be binary.
        If a tuple of ints, the ints are assumed to be the size of each variable. If a tuple
        of pairs of ints, each pair represents the min and max alphabet size of each variable.

    Returns
    -------
    dist : Distribution
        A distribution with variable sizes.
    """
    try:
        len(alphabets)
        try:
            len(alphabets[0])
        except TypeError:
            alphabets = tuple((alpha, alpha) for alpha in alphabets)
    except TypeError:
        alphabets = ((2, 2),)*alphabets

    alphabets = [draw(integers(*alpha)) for alpha in alphabets]

    pmf = draw(arrays(np.float, shape=alphabets, elements=floats(0, 1)))

    assume(pmf.sum() > 0)

    if nondegenerate:
        axes = set(range(len(alphabets)))
        for axis, _ in enumerate(alphabets):
            assume(np.sum(pmf.sum(axis=tuple(axes-set([axis]))) > 0) > 1)

    pmf /= pmf.sum()<commit_msg>make nondegenerate test more accurate<commit_after>"""
Utilities related to testing.
"""

from __future__ import division

from boltons.iterutils import pairwise

import numpy as np

from hypothesis import assume
from hypothesis.extra.numpy import arrays
from hypothesis.strategies import composite, floats, integers, lists, tuples

from .. import Distribution

__all__ = ['distributions',
           'distribution_structures',
           'markov_chains',
          ]

@composite
def distributions(draw, alphabets=(2, 2, 2), nondegenerate=False):
    """
    Generate distributions for use with hypothesis.

    Parameters
    ----------
    draw : function
        A sampling function passed in by hypothesis.
    alphabets : int, tuple of ints, tuple of pairs of ints
        If an int, it is the length of the chain and each variable is assumed to be binary.
        If a tuple of ints, the ints are assumed to be the size of each variable. If a tuple
        of pairs of ints, each pair represents the min and max alphabet size of each variable.

    Returns
    -------
    dist : Distribution
        A distribution with variable sizes.
    """
    try:
        len(alphabets)
        try:
            len(alphabets[0])
        except TypeError:
            alphabets = tuple((alpha, alpha) for alpha in alphabets)
    except TypeError:
        alphabets = ((2, 2),)*alphabets

    alphabets = [draw(integers(*alpha)) for alpha in alphabets]

    pmf = draw(arrays(np.float, shape=alphabets, elements=floats(0, 1)))

    assume(pmf.sum() > 0)

    if nondegenerate:
        axes = set(range(len(alphabets)))
        for axis, _ in enumerate(alphabets):
            assume(np.sum(pmf.sum(axis=tuple(axes-set([axis]))) > 1e-6) > 1)

    pmf /= pmf.sum()
{'fraction_non_alphanumeric': 0.09607112130771436, 'fraction_numerical': 0.00659592773157442, 'mean_word_length': 2.6638655462184873, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3785780', 'n_tokens_mistral': 1048, 'n_tokens_neox': 984, 'n_words': 359}