<commit_before>from collections import OrderedDict

from pytest import fixture

from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import _validate_assignment
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import _validate_format
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import _validate_plan_base
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import extract_actions_unique_topics
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import get_assignment_map
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import get_plan
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import get_reduced_proposed_plan
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import validate_plan


@fixture
def orig_assignment():
    return OrderedDict([
        ((u'T1', 1), [2, 1]),
        ((u'T0', 0), [0, 1]),
        ((u'T0', 1), [1, 2]),
        ((u'T1', 0), [0, 1]),
        ((u'T2', 0), [3, 1]),
    ])


@fixture
def new_assignment():
    return OrderedDict([
        ((u'T0', 0), [2, 0]),
        ((u'T1', 1), [2, 3]),
        ((u'T0', 1), [2, 1]),
        ((u'T1', 0), [0, 1]),
        ((u'T2', 0), [1, 3]),
    ])


def test_reduced_proposed_plan_no_change(orig_assignment):
    # Provide same assignment
    proposed_assignment = get_reduced_proposed_plan(
        original_assignment=orig_assignment,
        new_assignment=orig_assignment,
        max_partition_movements=1,
        max_leader_only_changes=1,
    )

    # Verify no proposed plan
    assert proposed_assignment == {}


def test_extract_actions_unique_topics_limited_actions():
    proposed_assignment = [
        ((u'T0', 0), [2, 0], 1),
        ((u'T0', 1), [2, 1], 1),
        ((u'T1', 0), [0, 1], 1),
        ((u'T2', 0), [1, 3], 1),
    ]
    red_proposed_assignment = extract_actions_unique_topics(
        proposed_assignment,
        3,
    )

    assert len(red_proposed_assignment) == 3
    topics = [action[0][0] for action in red_proposed_assignment]
    # Verify that all actions have unique topic
    assert sorted(topics) == sorted(list(set(topics)))


def test_extract_actions_partition_movement_no_action():
    # In case max-allowed partition-movements is less than replication-factor
    # there is a possibilty it will never converge
    proposed_assignment = [
        ((u'T0', 0), [2, 0], 2),
        ((u'T0', 1), [2, 1], 2),
        ((u'T1', 0), [0, 1], 2),
        ((u'T2', 0), [1, 3], 2),
    ]
    red_proposed_assignment = extract_actions_unique_topics(
        proposed_assignment,
        1,
    )

    # All actions have minimum of 2 movements
    # so reduced proposed-plan is empty
    assert red_proposed_assignment == []


def test_extract_actions_partition_movements_only():
    proposed_assignment = [
        ((u'T0', 0), [2, 0], 2),
        ((u'T0', 1), [2, 1], 1),
        ((u'T1', 0), [0, 1], 2),
        ((u'T2', 0), [1, 3], 1),
    ]
    red_proposed_assignment = extract_actions_unique_topics(
        proposed_assignment,
        10,
    )<commit_msg>Minor spell check<commit_after>from collections import OrderedDict

from pytest import fixture

from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import _validate_assignment
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import _validate_format
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import _validate_plan_base
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import extract_actions_unique_topics
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import get_assignment_map
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import get_plan
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import get_reduced_proposed_plan
from yelp_kafka_tool.kafka_cluster_manager.cluster_info.util import validate_plan


@fixture
def orig_assignment():
    return OrderedDict([
        ((u'T1', 1), [2, 1]),
        ((u'T0', 0), [0, 1]),
        ((u'T0', 1), [1, 2]),
        ((u'T1', 0), [0, 1]),
        ((u'T2', 0), [3, 1]),
    ])


@fixture
def new_assignment():
    return OrderedDict([
        ((u'T0', 0), [2, 0]),
        ((u'T1', 1), [2, 3]),
        ((u'T0', 1), [2, 1]),
        ((u'T1', 0), [0, 1]),
        ((u'T2', 0), [1, 3]),
    ])


def test_reduced_proposed_plan_no_change(orig_assignment):
    # Provide same assignment
    proposed_assignment = get_reduced_proposed_plan(
        original_assignment=orig_assignment,
        new_assignment=orig_assignment,
        max_partition_movements=1,
        max_leader_only_changes=1,
    )

    # Verify no proposed plan
    assert proposed_assignment == {}


def test_extract_actions_unique_topics_limited_actions():
    proposed_assignment = [
        ((u'T0', 0), [2, 0], 1),
        ((u'T0', 1), [2, 1], 1),
        ((u'T1', 0), [0, 1], 1),
        ((u'T2', 0), [1, 3], 1),
    ]
    red_proposed_assignment = extract_actions_unique_topics(
        proposed_assignment,
        3,
    )

    assert len(red_proposed_assignment) == 3
    topics = [action[0][0] for action in red_proposed_assignment]
    # Verify that all actions have unique topic
    assert sorted(topics) == sorted(list(set(topics)))


def test_extract_actions_partition_movement_no_action():
    # In case max-allowed partition-movements is less than replication-factor
    # there is a possibility it will never converge
    proposed_assignment = [
        ((u'T0', 0), [2, 0], 2),
        ((u'T0', 1), [2, 1], 2),
        ((u'T1', 0), [0, 1], 2),
        ((u'T2', 0), [1, 3], 2),
    ]
    red_proposed_assignment = extract_actions_unique_topics(
        proposed_assignment,
        1,
    )

    # All actions have minimum of 2 movements
    # so reduced proposed-plan is empty
    assert red_proposed_assignment == []


def test_extract_actions_partition_movements_only():
    proposed_assignment = [
        ((u'T0', 0), [2, 0], 2),
        ((u'T0', 1), [2, 1], 1),
        ((u'T1', 0), [0, 1], 2),
        ((u'T2', 0), [1, 3], 1),
    ]
    red_proposed_assignment = extract_actions_unique_topics(
        proposed_assignment,
        10,
    )
{'fraction_non_alphanumeric': 0.13252815407213972, 'fraction_numerical': 0.03590664272890485, 'mean_word_length': 3.430947216196674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4926362', 'n_tokens_mistral': 2304, 'n_tokens_neox': 2160, 'n_words': 476}