<commit_before>from __future__ import absolute_import, division, print_function, unicode_literals

from itertools import chain

from nose.tools import eq_, raises
from six.moves import xrange

from smarkets.streaming_api.framing import (
    frame_decode_all, frame_encode, IncompleteULEB128, uleb128_decode, uleb128_encode,
)


test_data = (
    (0x00000000, b'\x00'),
    (0x0000007F, b'\x7F'),
    (0x00000080, b'\x80\x01'),
    (624485, b'\xE5\x8E\x26'),
    (268435202, b'\x82\xFE\xFF\x7F'),
)


def test_dumps():
    for value, string in test_data:
        yield check_dumps, value, string


def check_dumps(value, string):
    eq_(uleb128_encode(value), string)


def test_loads():
    for value, string in test_data:
        yield check_loads, string, value


def check_loads(string, value):
    eq_(uleb128_decode(string), (value, len(string)))


def test_loads_and_dumps_are_consistent():
    for i in chain(
        xrange(2 ** 20),
        xrange(2 ** 20, 2 ** 26, 3333),
        xrange(2 ** 26, 2 ** 32, 77777),
    ):
        byte_dump = uleb128_encode(i)
        eq_(uleb128_decode(byte_dump), (i, len(byte_dump)))


@raises(ValueError)
def test_uleb128_encode_fails_on_negative_number():
    uleb128_encode(-1)


def test_uleb128_decode_fails_on_invalid_input():
    data = uleb128_encode(12345678)

    for i in xrange(len(data)):<commit_msg>Restrict the test's input set a bit to make it faster<commit_after>from __future__ import absolute_import, division, print_function, unicode_literals

from itertools import chain

from nose.tools import eq_, raises
from six.moves import xrange

from smarkets.streaming_api.framing import (
    frame_decode_all, frame_encode, IncompleteULEB128, uleb128_decode, uleb128_encode,
)


test_data = (
    (0x00000000, b'\x00'),
    (0x0000007F, b'\x7F'),
    (0x00000080, b'\x80\x01'),
    (624485, b'\xE5\x8E\x26'),
    (268435202, b'\x82\xFE\xFF\x7F'),
)


def test_dumps():
    for value, string in test_data:
        yield check_dumps, value, string


def check_dumps(value, string):
    eq_(uleb128_encode(value), string)


def test_loads():
    for value, string in test_data:
        yield check_loads, string, value


def check_loads(string, value):
    eq_(uleb128_decode(string), (value, len(string)))


def test_loads_and_dumps_are_consistent():
    for i in chain(
        xrange(2 ** 18),
        xrange(2 ** 20, 2 ** 26, 33333),
        xrange(2 ** 26, 2 ** 32, 777777),
    ):
        byte_dump = uleb128_encode(i)
        eq_(uleb128_decode(byte_dump), (i, len(byte_dump)))


@raises(ValueError)
def test_uleb128_encode_fails_on_negative_number():
    uleb128_encode(-1)


def test_uleb128_decode_fails_on_invalid_input():
    data = uleb128_encode(12345678)

    for i in xrange(len(data)):
{'fraction_non_alphanumeric': 0.11702898550724637, 'fraction_numerical': 0.08840579710144927, 'mean_word_length': 3.992766726943942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5094074', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1022, 'n_words': 223}