<commit_before>from common import *   # NOQA
from lib.aws import AmazonWebServices
import pytest
from threading import Thread


DO_ACCESSKEY = os.environ.get('DO_ACCESSKEY', "None")
AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
AWS_REGION = os.environ.get("AWS_REGION")
AWS_SUBNET = os.environ.get("AWS_SUBNET")
AWS_VPC = os.environ.get("AWS_VPC")
AWS_SG = os.environ.get("AWS_SG")
AWS_ZONE = os.environ.get("AWS_ZONE")
AWS_IAM_PROFILE = os.environ.get("AWS_IAM_PROFILE", "")
AZURE_SUBSCRIPTION_ID = os.environ.get("AZURE_SUBSCRIPTION_ID")
AZURE_CLIENT_ID = os.environ.get("AZURE_CLIENT_ID")
AZURE_CLIENT_SECRET = os.environ.get("AZURE_CLIENT_SECRET")

engine_install_url = "https://releases.rancher.com/install-docker/17.03.sh"
rke_config = {"authentication": {"type": "authnConfig", "strategy": "x509"},
              "ignoreDockerVersion": False,
              "network": {"type": "networkConfig", "plugin": "canal"},
              "type": "rancherKubernetesEngineConfig"
              }

rke_config_aws_provider = {"authentication": {"type": "authnConfig",
                                              "strategy": "x509"},
                           "ignoreDockerVersion": False,
                           "network": {"type": "networkConfig",
                                       "plugin": "canal"},
                           "type": "rancherKubernetesEngineConfig",
                           "cloudProvider": {"name": "aws",
                                             "type": "cloudProvider",
                                             "awsCloudProvider":
                                                 {"type": "awsCloudProvider"}}
                           }

if_stress_enabled = pytest.mark.skipif(
    not os.environ.get('RANCHER_STRESS_TEST_WORKER_COUNT'),
    reason='Stress test not enabled')

worker_count = int(os.environ.get('RANCHER_STRESS_TEST_WORKER_COUNT', 1))
RANCHER_CLEANUP_CLUSTER = os.environ.get('RANCHER_CLEANUP_CLUSTER', "True")


def test_rke_az_host_1(node_template_az):
    validate_rke_dm_host_1(node_template_az, rke_config)


def test_rke_az_host_2(node_template_az):
    validate_rke_dm_host_2(node_template_az, rke_config)


def test_rke_az_host_3(node_template_az):
    validate_rke_dm_host_3(node_template_az, rke_config)


def test_rke_az_host_4(node_template_az):
    validate_rke_dm_host_4(node_template_az, rke_config)


def test_rke_do_host_1(node_template_do):
    validate_rke_dm_host_1(node_template_do, rke_config)


def test_rke_do_host_2(node_template_do):
    validate_rke_dm_host_2(node_template_do, rke_config)


def test_rke_do_host_3(node_template_do):
    validate_rke_dm_host_3(node_template_do, rke_config)


def test_rke_do_host_4(node_template_do):
    validate_rke_dm_host_4(node_template_do, rke_config)


def test_rke_ec2_host_1(node_template_ec2):
    validate_rke_dm_host_1(node_template_ec2, rke_config)


def test_rke_ec2_host_2(node_template_ec2):
    validate_rke_dm_host_2(node_template_ec2, rke_config)


def test_rke_ec2_host_3(node_template_ec2):
    validate_rke_dm_host_3(node_template_ec2, rke_config)


def test_rke_ec2_host_with_aws_provider_1(node_template_ec2_with_provider):
    validate_rke_dm_host_1(node_template_ec2_with_provider,
                           rke_config_aws_provider)


def test_rke_ec2_host_with_aws_provider_2(node_template_ec2_with_provider):
    validate_rke_dm_host_2(node_template_ec2_with_provider,
                           rke_config_aws_provider)


def test_rke_ec2_host_4(node_template_ec2):
    validate_rke_dm_host_4(node_template_ec2, rke_config)


def test_rke_custom_host_1():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            1, random_test_name("testcustom"))
    node_roles = ["worker", "controlplane", "etcd"]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles,
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_2():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            5, random_test_name("testcustom"))
    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
        i += 1
    cluster = validate_cluster(client, cluster)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_3():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            8, random_test_name("testcustom"))
    node_roles = [
        ["controlplane"], ["controlplane"],
        ["etcd"], ["etcd"], ["etcd"],
        ["worker"], ["worker"], ["worker"]
    ]
    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
        i += 1
    cluster = validate_cluster(client, cluster)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_4():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            8, random_test_name("testcustom"))
    node_roles = [
        {"roles": ["controlplane"],
         "nodes":[aws_nodes[0], aws_nodes[1]]},
        {"roles": ["etcd"],
         "nodes": [aws_nodes[2], aws_nodes[3], aws_nodes[4]]},
        {"roles": ["worker"],
         "nodes": [aws_nodes[5], aws_nodes[6], aws_nodes[7]]}
    ]
    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    delay = 120
    host_threads = []
    for node_role in node_roles:
        host_thread = Thread(target=register_host_after_delay,
                             args=(client, cluster, node_role, delay))
        host_threads.append(host_thread)
        host_thread.start()
        time.sleep(30)
    for host_thread in host_threads:
        host_thread.join()
    cluster = validate_cluster(client, cluster,
                               check_intermediate_state=False)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


@if_stress_enabled
def test_rke_custom_host_stress():
    aws_nodes = AmazonWebServices().create_multiple_nodes(
        worker_count + 4, random_test_name("teststress"))
    node_roles = [["controlplane"], ["etcd"], ["etcd"], ["etcd"]]
    worker_role = ["worker"]
    for int in range(0, worker_count):
        node_roles.append(worker_role)
    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
        i += 1
    cluster = validate_cluster(client, cluster)


def test_rke_custom_host_etcd_plane_changes():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            7, random_test_name("testcustom"))
    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for i in range(0, 5):
        aws_node = aws_nodes[i]
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    etcd_nodes = get_role_nodes(cluster, "etcd")
    assert len(etcd_nodes) == 1

    # Add 1 more etcd node
    aws_node = aws_nodes[5]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)
    wait_for_cluster_node_count(client, cluster, 6)
    validate_cluster(client, cluster, intermediate_state="updating")

    # Add 1 more etcd node
    aws_node = aws_nodes[6]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)
    wait_for_cluster_node_count(client, cluster, 7)
    validate_cluster(client, cluster, intermediate_state="updating")

    # Delete the first etcd node
    client.delete(etcd_nodes[0])
    validate_cluster(client, cluster, intermediate_state="updating")

    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_etcd_plane_changes_1():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            7, random_test_name("testcustom"))
    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for i in range(0, 5):
        aws_node = aws_nodes[i]
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster,
                                             node_roles[i], aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    etcd_nodes = get_role_nodes(cluster, "etcd")
    assert len(etcd_nodes) == 1

    # Add 2 more etcd node
    aws_node = aws_nodes[5]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)

    aws_node = aws_nodes[6]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)

    wait_for_cluster_node_count(client, cluster, 7)
    validate_cluster(client, cluster, intermediate_state="updating")
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_control_plane_changes():
    aws_nodes = \
        aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            6, random_test_name("testcustom"))
    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for i in range(0, 5):
        aws_node = aws_nodes[i]
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster,
                                             node_roles[i], aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    control_nodes = get_role_nodes(cluster, "control")
    assert len(control_nodes) == 1

    # Add 1 more control node
    aws_node = aws_nodes[5]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["controlplane"],
                                                      aws_node)
    aws_node.execute_command(docker_run_cmd)
    wait_for_cluster_node_count(client, cluster, 6)<commit_msg>Added azure w/ cloud provider provisioning<commit_after>from common import *   # NOQA
from lib.aws import AmazonWebServices
import pytest
from threading import Thread


DO_ACCESSKEY = os.environ.get('DO_ACCESSKEY', "None")
AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
AWS_REGION = os.environ.get("AWS_REGION")
AWS_SUBNET = os.environ.get("AWS_SUBNET")
AWS_VPC = os.environ.get("AWS_VPC")
AWS_SG = os.environ.get("AWS_SG")
AWS_ZONE = os.environ.get("AWS_ZONE")
AWS_IAM_PROFILE = os.environ.get("AWS_IAM_PROFILE", "")
AZURE_SUBSCRIPTION_ID = os.environ.get("AZURE_SUBSCRIPTION_ID")
AZURE_CLIENT_ID = os.environ.get("AZURE_CLIENT_ID")
AZURE_CLIENT_SECRET = os.environ.get("AZURE_CLIENT_SECRET")
AZURE_TENANT_ID = os.environ.get("AZURE_TENANT_ID")

engine_install_url = "https://releases.rancher.com/install-docker/17.03.sh"
rke_config = {"authentication": {"type": "authnConfig", "strategy": "x509"},
              "ignoreDockerVersion": False,
              "network": {"type": "networkConfig", "plugin": "canal"},
              "type": "rancherKubernetesEngineConfig"
              }

rke_config_aws_provider = {"authentication": {"type": "authnConfig",
                                              "strategy": "x509"},
                           "ignoreDockerVersion": False,
                           "network": {"type": "networkConfig",
                                       "plugin": "canal"},
                           "type": "rancherKubernetesEngineConfig",
                           "cloudProvider": {"name": "aws",
                                             "type": "cloudProvider",
                                             "awsCloudProvider":
                                                 {"type": "awsCloudProvider"}}
                           }


rke_config_azure_provider = {"authentication": {"type": "authnConfig",
                                                "strategy": "x509"},
                             "ignoreDockerVersion": False,
                             "network": {"type": "networkConfig",
                                         "plugin": "canal"},
                             "type": "rancherKubernetesEngineConfig",
                             "cloudProvider": {
                             "type": "cloudProvider",
                             "name": "azure",
                             "azureCloudProvider": {
                               "aadClientId": AZURE_CLIENT_ID,
                               "aadClientSecret": AZURE_CLIENT_SECRET,
                               "subscriptionId": AZURE_SUBSCRIPTION_ID,
                               "tenantId": AZURE_TENANT_ID}}
                             }

if_stress_enabled = pytest.mark.skipif(
    not os.environ.get('RANCHER_STRESS_TEST_WORKER_COUNT'),
    reason='Stress test not enabled')

worker_count = int(os.environ.get('RANCHER_STRESS_TEST_WORKER_COUNT', 1))
RANCHER_CLEANUP_CLUSTER = os.environ.get('RANCHER_CLEANUP_CLUSTER', "True")


def test_rke_az_host_1(node_template_az):
    validate_rke_dm_host_1(node_template_az, rke_config)


def test_rke_az_host_2(node_template_az):
    validate_rke_dm_host_2(node_template_az, rke_config)


def test_rke_az_host_3(node_template_az):
    validate_rke_dm_host_3(node_template_az, rke_config)


def test_rke_az_host_4(node_template_az):
    validate_rke_dm_host_4(node_template_az, rke_config)


def test_rke_az_host_with_provider_1(node_template_az):
    validate_rke_dm_host_1(node_template_az, rke_config_azure_provider)


def test_rke_az_host_with_provider_2(node_template_az):
    validate_rke_dm_host_2(node_template_az, rke_config_azure_provider)


def test_rke_do_host_1(node_template_do):
    validate_rke_dm_host_1(node_template_do, rke_config)


def test_rke_do_host_2(node_template_do):
    validate_rke_dm_host_2(node_template_do, rke_config)


def test_rke_do_host_3(node_template_do):
    validate_rke_dm_host_3(node_template_do, rke_config)


def test_rke_do_host_4(node_template_do):
    validate_rke_dm_host_4(node_template_do, rke_config)


def test_rke_ec2_host_1(node_template_ec2):
    validate_rke_dm_host_1(node_template_ec2, rke_config)


def test_rke_ec2_host_2(node_template_ec2):
    validate_rke_dm_host_2(node_template_ec2, rke_config)


def test_rke_ec2_host_3(node_template_ec2):
    validate_rke_dm_host_3(node_template_ec2, rke_config)


def test_rke_ec2_host_with_aws_provider_1(node_template_ec2_with_provider):
    validate_rke_dm_host_1(node_template_ec2_with_provider,
                           rke_config_aws_provider)


def test_rke_ec2_host_with_aws_provider_2(node_template_ec2_with_provider):
    validate_rke_dm_host_2(node_template_ec2_with_provider,
                           rke_config_aws_provider)


def test_rke_ec2_host_4(node_template_ec2):
    validate_rke_dm_host_4(node_template_ec2, rke_config)


def test_rke_custom_host_1():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            1, random_test_name("testcustom"))
    node_roles = ["worker", "controlplane", "etcd"]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles,
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_2():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            5, random_test_name("testcustom"))
    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
        i += 1
    cluster = validate_cluster(client, cluster)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_3():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            8, random_test_name("testcustom"))
    node_roles = [
        ["controlplane"], ["controlplane"],
        ["etcd"], ["etcd"], ["etcd"],
        ["worker"], ["worker"], ["worker"]
    ]
    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
        i += 1
    cluster = validate_cluster(client, cluster)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_4():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            8, random_test_name("testcustom"))
    node_roles = [
        {"roles": ["controlplane"],
         "nodes":[aws_nodes[0], aws_nodes[1]]},
        {"roles": ["etcd"],
         "nodes": [aws_nodes[2], aws_nodes[3], aws_nodes[4]]},
        {"roles": ["worker"],
         "nodes": [aws_nodes[5], aws_nodes[6], aws_nodes[7]]}
    ]
    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    delay = 120
    host_threads = []
    for node_role in node_roles:
        host_thread = Thread(target=register_host_after_delay,
                             args=(client, cluster, node_role, delay))
        host_threads.append(host_thread)
        host_thread.start()
        time.sleep(30)
    for host_thread in host_threads:
        host_thread.join()
    cluster = validate_cluster(client, cluster,
                               check_intermediate_state=False)
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


@if_stress_enabled
def test_rke_custom_host_stress():
    aws_nodes = AmazonWebServices().create_multiple_nodes(
        worker_count + 4, random_test_name("teststress"))

    node_roles = [["controlplane"], ["etcd"], ["etcd"], ["etcd"]]
    worker_role = ["worker"]
    for int in range(0, worker_count):
        node_roles.append(worker_role)
    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
        i += 1
    cluster = validate_cluster(client, cluster)


def test_rke_custom_host_etcd_plane_changes():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            7, random_test_name("testcustom"))
    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for i in range(0, 5):
        aws_node = aws_nodes[i]
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, node_roles[i],
                                             aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    etcd_nodes = get_role_nodes(cluster, "etcd")
    assert len(etcd_nodes) == 1

    # Add 1 more etcd node
    aws_node = aws_nodes[5]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)
    wait_for_cluster_node_count(client, cluster, 6)
    validate_cluster(client, cluster, intermediate_state="updating")

    # Add 1 more etcd node
    aws_node = aws_nodes[6]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)
    wait_for_cluster_node_count(client, cluster, 7)
    validate_cluster(client, cluster, intermediate_state="updating")

    # Delete the first etcd node
    client.delete(etcd_nodes[0])
    validate_cluster(client, cluster, intermediate_state="updating")

    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_etcd_plane_changes_1():
    aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            7, random_test_name("testcustom"))
    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for i in range(0, 5):
        aws_node = aws_nodes[i]
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster,
                                             node_roles[i], aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    etcd_nodes = get_role_nodes(cluster, "etcd")
    assert len(etcd_nodes) == 1

    # Add 2 more etcd node
    aws_node = aws_nodes[5]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)

    aws_node = aws_nodes[6]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["etcd"], aws_node)
    aws_node.execute_command(docker_run_cmd)

    wait_for_cluster_node_count(client, cluster, 7)
    validate_cluster(client, cluster, intermediate_state="updating")
    if RANCHER_CLEANUP_CLUSTER == "True":
        delete_cluster(client, cluster)
        delete_node(aws_nodes)


def test_rke_custom_host_control_plane_changes():
    aws_nodes = \
        aws_nodes = \
        AmazonWebServices().create_multiple_nodes(
            6, random_test_name("testcustom"))

    node_roles = [["controlplane"], ["etcd"],
                  ["worker"], ["worker"], ["worker"]]

    client = get_admin_client()
    cluster = client.create_cluster(name=random_name(),
                                    driver="rancherKubernetesEngine",
                                    rancherKubernetesEngineConfig=rke_config)
    assert cluster.state == "active"
    i = 0
    for i in range(0, 5):
        aws_node = aws_nodes[i]
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster,
                                             node_roles[i], aws_node)
        aws_node.execute_command(docker_run_cmd)
    cluster = validate_cluster(client, cluster)
    control_nodes = get_role_nodes(cluster, "control")
    assert len(control_nodes) == 1

    # Add 1 more control node
    aws_node = aws_nodes[5]
    docker_run_cmd = get_custom_host_registration_cmd(client, cluster,
                                                      ["controlplane"],
                                                      aws_node)
    aws_node.execute_command(docker_run_cmd)
    wait_for_cluster_node_count(client, cluster, 6)
{'fraction_non_alphanumeric': 0.09574318628828322, 'fraction_numerical': 0.008394211857263276, 'mean_word_length': 2.448764534883721, 'pattern_counts': {'":': 82, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 106, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3304918', 'n_tokens_mistral': 8857, 'n_tokens_neox': 8684, 'n_words': 1295}