<commit_before>import json

from json_parser import load_string, load_string_faster


def tests():
    s = r'{ "foo": "bar", "yo": [{ "hi": "there\"blah" }, 1000] }'
    s2 = r'{ "one": { "two": [{ "three": { "four": null }}, false ], "five": 5 }}'

    print
    print 'JSON:', s
    print 'PYTHON:', load_string_faster(s)
    assert json.loads(s) == load_string(s) == load_string_faster(s)

    print
    print 'JSON:', s2
    print 'PYTHON:', load_string_faster(s2)
    assert json.loads(s2) == load_string(s2) == load_string_faster(s2)

    print
    print 'Tests pass!'
<commit_msg>Update tests for JSON parser.<commit_after>import json

from json_parser import load_string, load_string_faster


def tests():
    test_cases = [
        r'{ "foo": "bar", "yo": [{ "hi": "there\"blah" }, 1000] }',
        r'{ "one": { "two": [{ "three": { "four": null }}, false ], "five": 5 }}',
        r'{ "foo": "bar", "baz": null }',
    ]

    for json_string in test_cases:
        print
        print 'JSON:', json_string
        print 'PYTHON:', load_string_faster(json_string)
        assert json.loads(json_string) == load_string(json_string) == load_string_faster(json_string)

    print
    print 'Tests pass!'

{'fraction_non_alphanumeric': 0.17671345995045418, 'fraction_numerical': 0.013212221304706853, 'mean_word_length': 3.3440860215053765, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1507874', 'n_tokens_mistral': 452, 'n_tokens_neox': 428, 'n_words': 107}