<commit_before>import random

import pytest

from .. import hash_schema


def generate_schemas():
    yield 4
    yield 3.14
    yield 'foo'
    yield None
    yield {'a': 1, 'b': 2}
    yield [1, 2, 3]
    yield {1, 2, 3}
    yield (1, 2, 3)


def scramble(val):
    if isinstance(val, dict):
        L = [(k, scramble(v)) for k, v in val.items()]
        random.shuffle(L)
        return dict(L)
    elif isinstance(val, set):
        L = [scramble(v) for v in val]
        random.shuffle(L)<commit_msg>TST: add nested schema to hash test<commit_after>import random

import pytest

from .. import hash_schema


def generate_schemas():
    yield 4
    yield 3.14
    yield 'foo'
    yield None
    yield {'a': 1, 'b': 2}
    yield [1, 2, 3]
    yield {1, 2, 3}
    yield (1, 2, 3)
    yield {
        'a': {
            'key': [1, 2, 3],
            'val': {'a', 'b', 'c'}
            },
        'b': [{}, {'name': 'bob', 'age': 31}],
        'c': (1, 2, 3)
    }


def scramble(val):
    if isinstance(val, dict):
        L = [(k, scramble(v)) for k, v in val.items()]
        random.shuffle(L)
        return dict(L)
    elif isinstance(val, set):
        L = [scramble(v) for v in val]
        random.shuffle(L)
{'fraction_non_alphanumeric': 0.1588477366255144, 'fraction_numerical': 0.031275720164609055, 'mean_word_length': 1.9877149877149878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4108825', 'n_tokens_mistral': 485, 'n_tokens_neox': 444, 'n_words': 144}