<commit_before>import numpy
import theano
from numpy.testing import assert_equal, assert_, assert_raises
from theano.scalar.basic import all_types
from pylearn2.format.target_format import OneHotFormatter, compressed_one_hot


def test_one_hot_formatter_simple():
    def check_one_hot_formatter(seed, max_labels, dtype, ncases):
        rng = numpy.random.RandomState(seed)
        fmt = OneHotFormatter(max_labels=max_labels, dtype=dtype)
        integer_labels = rng.random_integers(0, max_labels - 1, size=ncases)
        one_hot_labels = fmt.format(integer_labels)
        assert len(list(zip(*one_hot_labels.nonzero()))) == ncases
        for case, label in enumerate(integer_labels):
            assert one_hot_labels[case, label] == 1
    rng = numpy.random.RandomState(0)
    for seed, dtype in enumerate(all_types):
        yield (check_one_hot_formatter, seed, rng.random_integers(1, 30),
               dtype, rng.random_integers(1, 100))
    fmt = OneHotFormatter(max_labels=10)
    assert fmt.format(numpy.zeros((1, 1), dtype='uint8')).shape == (1, 1, 10)


def test_one_hot_formatter_symbolic():
    def check_one_hot_formatter_symbolic(seed, max_labels, dtype, ncases):
        rng = numpy.random.RandomState(seed)
        fmt = OneHotFormatter(max_labels=max_labels, dtype=dtype)
        integer_labels = rng.random_integers(0, max_labels - 1, size=ncases)
        x = theano.tensor.vector(dtype='int64')
        y = fmt.theano_expr(x)
        f = theano.function([x], y)
        one_hot_labels = f(integer_labels)
        assert len(list(zip(*one_hot_labels.nonzero()))) == ncases
        for case, label in enumerate(integer_labels):
            assert one_hot_labels[case, label] == 1

    rng = numpy.random.RandomState(0)
    for seed, dtype in enumerate(all_types):
        yield (check_one_hot_formatter_symbolic, seed,
               rng.random_integers(1, 30), dtype,
               rng.random_integers(1, 100))


def test_dtype_errors():
    # Try to call theano_expr with a bad label dtype.
    raised = False
    fmt = OneHotFormatter(max_labels=50)
    try:
        fmt.theano_expr(theano.tensor.vector(dtype=theano.config.floatX))
    except TypeError:<commit_msg>Make sure dtype of formatted data is as expected<commit_after>import numpy
import theano
from numpy.testing import assert_equal, assert_, assert_raises
from theano.scalar.basic import all_types
from pylearn2.format.target_format import OneHotFormatter, compressed_one_hot


def test_one_hot_formatter_simple():
    def check_one_hot_formatter(seed, max_labels, dtype, ncases):
        rng = numpy.random.RandomState(seed)
        fmt = OneHotFormatter(max_labels=max_labels, dtype=dtype)
        integer_labels = rng.random_integers(0, max_labels - 1, size=ncases)
        one_hot_labels = fmt.format(integer_labels)
        assert len(list(zip(*one_hot_labels.nonzero()))) == ncases
        assert one_hot_labels.dtype == dtype
        for case, label in enumerate(integer_labels):
            assert one_hot_labels[case, label] == 1
    rng = numpy.random.RandomState(0)
    for seed, dtype in enumerate(all_types):
        yield (check_one_hot_formatter, seed, rng.random_integers(1, 30),
               dtype, rng.random_integers(1, 100))
    fmt = OneHotFormatter(max_labels=10)
    assert fmt.format(numpy.zeros((1, 1), dtype='uint8')).shape == (1, 1, 10)


def test_one_hot_formatter_symbolic():
    def check_one_hot_formatter_symbolic(seed, max_labels, dtype, ncases):
        rng = numpy.random.RandomState(seed)
        fmt = OneHotFormatter(max_labels=max_labels, dtype=dtype)
        integer_labels = rng.random_integers(0, max_labels - 1, size=ncases)
        x = theano.tensor.vector(dtype='int64')
        y = fmt.theano_expr(x)
        f = theano.function([x], y)
        one_hot_labels = f(integer_labels)
        assert len(list(zip(*one_hot_labels.nonzero()))) == ncases
        assert one_hot_labels.dtype == dtype
        for case, label in enumerate(integer_labels):
            assert one_hot_labels[case, label] == 1

    rng = numpy.random.RandomState(0)
    for seed, dtype in enumerate(all_types):
        yield (check_one_hot_formatter_symbolic, seed,
               rng.random_integers(1, 30), dtype,
               rng.random_integers(1, 100))


def test_dtype_errors():
    # Try to call theano_expr with a bad label dtype.
    raised = False
    fmt = OneHotFormatter(max_labels=50)
    try:
        fmt.theano_expr(theano.tensor.vector(dtype=theano.config.floatX))
    except TypeError:
{'fraction_non_alphanumeric': 0.10188261351052048, 'fraction_numerical': 0.015946843853820596, 'mean_word_length': 3.5570131180625633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5889921', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1454, 'n_words': 305}