<commit_before>import sys
import unittest

import vdf
from collections import OrderedDict

u = str if sys.version_info >= (3,) else unicode


class BinaryVDF(unittest.TestCase):
    def test_BASE_INT(self):
        repr(vdf.BASE_INT())

    def test_simple(self):
        pairs = [
            ('a', 'test'),
            ('a2', b'\xff\xfe0\x041\x042\x043\x04'.decode('utf-16')),
            ('bb', 1),
            ('bb2', -500),
            ('ccc', 1.0),
            ('dddd', vdf.POINTER(1234)),
            ('fffff', vdf.COLOR(1234)),
            ('gggggg', vdf.UINT_64(1234)),
            ('hhhhhhh', vdf.INT_64(-1234)),
        ]

        data = OrderedDict(pairs)
        data['level1-1'] = OrderedDict(pairs)
        data['level1-1']['level2-1'] = OrderedDict(pairs)
        data['level1-1']['level2-2'] = OrderedDict(pairs)
        data['level1-2'] = OrderedDict(pairs)
<commit_msg>fix failing test after dd91fb1 change<commit_after>import sys
import unittest

import vdf
from collections import OrderedDict

u = str if sys.version_info >= (3,) else unicode


class BinaryVDF(unittest.TestCase):
    def test_BASE_INT(self):
        repr(vdf.BASE_INT())

    def test_simple(self):
        pairs = [
            ('a', 'test'),
            ('a2', b'\xd0\xb0\xd0\xb1\xd0\xb2\xd0\xb3'.decode('utf-8')),
            ('bb', 1),
            ('bb2', -500),
            ('ccc', 1.0),
            ('dddd', vdf.POINTER(1234)),
            ('fffff', vdf.COLOR(1234)),
            ('gggggg', vdf.UINT_64(1234)),
            ('hhhhhhh', vdf.INT_64(-1234)),
        ]

        data = OrderedDict(pairs)
        data['level1-1'] = OrderedDict(pairs)
        data['level1-1']['level2-1'] = OrderedDict(pairs)
        data['level1-1']['level2-2'] = OrderedDict(pairs)
        data['level1-2'] = OrderedDict(pairs)

{'fraction_non_alphanumeric': 0.17637271214642264, 'fraction_numerical': 0.059900166389351084, 'mean_word_length': 2.256317689530686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6056675', 'n_tokens_mistral': 696, 'n_tokens_neox': 636, 'n_words': 111}