<commit_before>from collections import defaultdict
import random
import string
from qclient.node_ring import NodeRing


def test_distribution():
    random_strings = [''.join(random.choice(string.ascii_uppercase) for _ in range(6)) for _ in range(60000)]
    node_weights = {'aaa': 1, 'bbb': 2, 'ccc': 1, 'ddd': 1, 'eee': 1}

    ring = NodeRing(node_weights.keys(), node_weights)
    distribution = defaultdict(lambda: 1)
    for s in random_strings:
        distribution[ring.get_node(s)] += 1

    assert 9000 < distribution['aaa'] < 11000
    assert 18000 < distribution['bbb'] < 22000
    assert 9000 < distribution['ccc'] < 11000
    assert 9000 < distribution['ddd'] < 11000
    assert 9000 < distribution['eee'] < 11000

    # Drop a node
    ring.remove_node('aaa')

    removed_distribution = defaultdict(lambda: 1)
    for s in random_strings:
        removed_distribution[ring.get_node(s)] += 1

    assert 22000 < removed_distribution['bbb'] < 26000
    assert 11000 < removed_distribution['ccc'] < 13000
    assert 11000 < removed_distribution['ddd'] < 13000
    assert 11000 < removed_distribution['eee'] < 13000

    # Add the node back again and check that the distribution goes back to previous
    ring.add_node('aaa', weight=1)

    added_distribution = defaultdict(lambda: 1)
    for s in random_strings:
        added_distribution[ring.get_node(s)] += 1

    assert distribution['aaa'] == added_distribution['aaa']
    assert distribution['bbb'] == added_distribution['bbb']
    assert distribution['ccc'] == added_distribution['ccc']
    assert distribution['ddd'] == added_distribution['ddd']
    assert distribution['eee'] == added_distribution['eee']
<commit_msg>Improved node ring tests a little<commit_after>from collections import defaultdict
import random
import string
from qclient.node_ring import NodeRing


def test_distribution():
    random_strings = [''.join(random.choice(string.ascii_uppercase) for _ in range(6)) for _ in range(60000)]
    node_weights = {'aaa': 1, 'bbb': 2, 'ccc': 1, 'ddd': 1, 'eee': 1}

    ring = NodeRing(node_weights.keys(), node_weights)
    distribution = defaultdict(list)
    for s in random_strings:
        distribution[ring.get_node(s)].append(s)

    assert 9000 < len(distribution['aaa']) < 11000
    assert 18000 < len(distribution['bbb']) < 22000
    assert 9000 < len(distribution['ccc']) < 11000
    assert 9000 < len(distribution['ddd']) < 11000
    assert 9000 < len(distribution['eee']) < 11000

    # Drop a node, the new distributions should be super sets of the previous.
    # Existing nodes should not move.
    ring.remove_node('aaa')

    removed_distribution = defaultdict(list)
    for s in random_strings:
        removed_distribution[ring.get_node(s)].append(s)

    assert 22000 < len(removed_distribution['bbb']) < 26000
    assert 11000 < len(removed_distribution['ccc']) < 13000
    assert 11000 < len(removed_distribution['ddd']) < 13000
    assert 11000 < len(removed_distribution['eee']) < 13000

    assert set(distribution['bbb']) <= set(removed_distribution['bbb'])
    assert set(distribution['ccc']) <= set(removed_distribution['ccc'])
    assert set(distribution['ddd']) <= set(removed_distribution['ddd'])
    assert set(distribution['eee']) <= set(removed_distribution['eee'])

    # Add the node back again and check that the distribution goes back to previous
    ring.add_node('aaa', weight=1)

    added_distribution = defaultdict(list)
    for s in random_strings:
        added_distribution[ring.get_node(s)].append(s)

    assert distribution['aaa'] == added_distribution['aaa']
    assert distribution['bbb'] == added_distribution['bbb']
    assert distribution['ccc'] == added_distribution['ccc']
    assert distribution['ddd'] == added_distribution['ddd']
    assert distribution['eee'] == added_distribution['eee']

{'fraction_non_alphanumeric': 0.12790394152962672, 'fraction_numerical': 0.05272774732445837, 'mean_word_length': 4.482117310443491, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7283599', 'n_tokens_mistral': 1355, 'n_tokens_neox': 1114, 'n_words': 294}