<commit_before>"""
test.tools

Tools that are useful for writing unit tests for InPUT.

:copyright: (c) 2013 by <NAME>.
:license: MIT. See LICENSE for details.
"""

import warnings

def assertVariability(f, iterations=10):
    """ Assert that f will generate at least one unique value. """
    assert iterations > 1, '1 iteration makes no sense!'
    values = []
    for i in range(iterations):
        value = f()
        if value not in values and len(values) > 0:
            return True     # Found a new value.
        else:
            values.append(value)
    msg = 'Only generated %s for %i iterations' % (value, iterations)
    raise AssertionError(msg)

def assertConstancy(f, iterations=10):
    """ Assert that f will only generate a constant value. """
    assert iterations > 1, '1 iteration makes no sense!'
    values = []
    for i in range(iterations):
        value = f()
        if value not in values and len(values) > 0:
            msg = 'Generated a unique value after %i iterations' % (iterations)
            raise AssertionError(msg)
        else:
            values.append(value)
    return True             # No variation after N iterations.


def assertMatchingArrayDimensions(sizes, array):
    """
    Assert that the array dimensions match the given sizes.
    The sizes are listed in order of nesting, which implies that the
    sub-arrays on each level must be uniformly sized.
    """
    size = sizes[0]
    if size != len(array):
        msg = '%i does not match length of %s' % (size, len(array))
        raise AssertionError(msg)
    if len(sizes) == 1:
        return True         # No more dimensions. Done.
    for element in array:
        # Check sub-arrays recursively.
        assertMatchingArrayDimensions(sizes[1:], element)
    return True


# Note that these two functions can be used in two different ways, simply
# by flipping the order of their arguments. The values can either be
# matched against expected values, or the other way around. Both are useful.

# TODO:
# These two still need to be replaced assert versions (or possibly removed).
def checkAnyValueMatches(values, expected):
    """
    Return whether any of the values match an expected value.
    The return value is True if and only if there exists some value v
    such that v is one of the expected values.
    """
    warnings.warn('use the assert version instead', DeprecationWarning)
    return any([v in expected for v in values])

def checkAllValuesMatch(values, expected):
    """
    Return whether all of the values match an expected value.
    The return value is True if and only if there does not exist some
    value v such that v is not one of the expected values.
    """
    warnings.warn('use the assert version instead', DeprecationWarning)
    return all([v in expected for v in values])


def assertGeneratesAny(f, expected, iterations=10):
    """ Assert that f generates at least one expected value. """
    for i in range(iterations):
        if f() in expected:
            return True
    msg = 'Generated none of %s after %i iterations' % (expected, iterations)
    raise AssertionError(msg)

def assertGeneratesAll(f, expected, iterations=None):
    """
    Assert that f generates all expected values.
    If given, iterations must be >= the number of expected values.
    """
    if iterations is None:
        iterations = len(expected) * 10
    else:
        msg = 'Must generate at least as many values as are expected'
        assert iterations > len(expected), msg

    expected = list(expected)
    for i in range(iterations):
        v = f()
        if v in expected:
            expected.remove(v)
        if len(expected) == 0:
            return True
    msg = '%s still missing after %i iterations' % (expected, iterations)
    raise AssertionError(msg)

def assertGeneratesOnly(f, expected, iterations=10):
    """ Assert only expected values are generated by f. """
    for i in range(iterations):<commit_msg>Test tools use a symbolic constant<commit_after>"""
test.tools

Tools that are useful for writing unit tests for InPUT.

:copyright: (c) 2013 by <NAME>.
:license: MIT. See LICENSE for details.
"""

import warnings

DEFAULT_ITERATIONS = 10

def assertVariability(f, iterations=DEFAULT_ITERATIONS):
    """ Assert that f will generate at least one unique value. """
    assert iterations > 1, '1 iteration makes no sense!'
    values = []
    for i in range(iterations):
        value = f()
        if value not in values and len(values) > 0:
            return True     # Found a new value.
        else:
            values.append(value)
    msg = 'Only generated %s for %i iterations' % (value, iterations)
    raise AssertionError(msg)

def assertConstancy(f, iterations=DEFAULT_ITERATIONS):
    """ Assert that f will only generate a constant value. """
    assert iterations > 1, '1 iteration makes no sense!'
    values = []
    for i in range(iterations):
        value = f()
        if value not in values and len(values) > 0:
            msg = 'Generated a unique value after %i iterations' % (iterations)
            raise AssertionError(msg)
        else:
            values.append(value)
    return True             # No variation after N iterations.


def assertMatchingArrayDimensions(sizes, array):
    """
    Assert that the array dimensions match the given sizes.
    The sizes are listed in order of nesting, which implies that the
    sub-arrays on each level must be uniformly sized.
    """
    size = sizes[0]
    if size != len(array):
        msg = '%i does not match length of %s' % (size, len(array))
        raise AssertionError(msg)
    if len(sizes) == 1:
        return True         # No more dimensions. Done.
    for element in array:
        # Check sub-arrays recursively.
        assertMatchingArrayDimensions(sizes[1:], element)
    return True


# Note that these two functions can be used in two different ways, simply
# by flipping the order of their arguments. The values can either be
# matched against expected values, or the other way around. Both are useful.

# TODO:
# These two still need to be replaced assert versions (or possibly removed).
def checkAnyValueMatches(values, expected):
    """
    Return whether any of the values match an expected value.
    The return value is True if and only if there exists some value v
    such that v is one of the expected values.
    """
    warnings.warn('use the assert version instead', DeprecationWarning)
    return any([v in expected for v in values])

def checkAllValuesMatch(values, expected):
    """
    Return whether all of the values match an expected value.
    The return value is True if and only if there does not exist some
    value v such that v is not one of the expected values.
    """
    warnings.warn('use the assert version instead', DeprecationWarning)
    return all([v in expected for v in values])


def assertGeneratesAny(f, expected, iterations=DEFAULT_ITERATIONS):
    """ Assert that f generates at least one expected value. """
    for i in range(iterations):
        if f() in expected:
            return True
    msg = 'Generated none of %s after %i iterations' % (expected, iterations)
    raise AssertionError(msg)

def assertGeneratesAll(f, expected, iterations=None):
    """
    Assert that f generates all expected values.
    If given, iterations must be >= the number of expected values.
    """
    if iterations is None:
        iterations = len(expected) * DEFAULT_ITERATIONS
    else:
        msg = 'Must generate at least as many values as are expected'
        assert iterations > len(expected), msg

    expected = list(expected)
    for i in range(iterations):
        v = f()
        if v in expected:
            expected.remove(v)
        if len(expected) == 0:
            return True
    msg = '%s still missing after %i iterations' % (expected, iterations)
    raise AssertionError(msg)

def assertGeneratesOnly(f, expected, iterations=DEFAULT_ITERATIONS):
    """ Assert only expected values are generated by f. """
    for i in range(iterations):
{'fraction_non_alphanumeric': 0.08190830235439901, 'fraction_numerical': 0.004956629491945477, 'mean_word_length': 2.81067044381492, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1593182', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2100, 'n_words': 968}