func ensureNewArgumentsWereProcessed(c *testContext) {
	log.Info("Ensuring new arguments are present on nodes in cluster...")
	conns := []conn{{ip: "127.0.0.1", port: "2223"}, {ip: "127.0.0.1", port: "2224"}}
	for name, val := range kubeletArgs {
		for _, conn := range conns {
			argString := fmt.Sprintf("%s=%s", name, val)
			c.makeSSHCallWithRetries(conn.ip, conn.port, fmt.Sprintf("ps -ef | grep -v 'ps -ef' | grep /usr/bin/kubelet | grep %s", argString), 5)
		}
	}
	for name, val := range apiServerArgs {
		for _, conn := range conns {
			argString := fmt.Sprintf("%s=%s", name, val)
			c.makeSSHCallWithRetries(conn.ip, conn.port, fmt.Sprintf("ps -ef | grep -v 'ps -ef' | grep kube-apiserver | grep %s", argString), 5)
		}
	}
	for _, conn := range conns {
		c.makeSSHCallWithFailureHandler(
			conn.ip,
			conn.port,
			"grep 'io.kubernetes.pod.name' /etc/docker/daemon.json",
			func() {
				c.sshAction(conn.ip, conn.port, "cat /etc/docker/daemon.json")
			},
			5)
	}
}
Check that arguments show up after being changed on the fly
package test
import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"text/template"
	"time"
	log "github.com/sirupsen/logrus"
	"github.com/stretchr/testify/require"
	"github.com/weaveworks/cluster-api-provider-existinginfra/apis/cluster.weave.works/v1alpha3"
	capeios "github.com/weaveworks/cluster-api-provider-existinginfra/pkg/apis/wksprovider/machine/os"
	clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1"
	"sigs.k8s.io/yaml"
)
const (
	footlooseConfig = `
cluster:
  name: centos-multimaster
  privateKey: cluster-key
machines:
- count: 3
  spec:
    image: quay.io/footloose/centos7:0.3.0
    name: node%d
    portMappings:
    - containerPort: 22
      hostPort: 2222
    - containerPort: 6443
      hostPort: 6443
    - containerPort: 30443
      hostPort: 30443
    - containerPort: 30080
      hostPort: 30080
    privileged: true
    volumes:
    - type: volume
      destination: /var/lib/docker
`
	testNamespace = `
apiVersion: v1
kind: Namespace
metadata:
  labels:
  name: test
`
	testPoolTemplate = `
apiVersion: v1
kind: Secret
metadata:
  name: ip-pool
  namespace: weavek8sops
type: Opaque
data:
  config: {{ .SecretData }}
`
	clusterctlConfig = `
providers:
  - name: "existinginfra"
    url: "file://{{ .HomeDir }}/local-repository/infrastructure-existinginfra/v0.1.0/infrastructure-components.yaml"
    type: "InfrastructureProvider"
`
)
var (
	apiServerArgs = map[string]string{"alsologtostderr": "true", "audit-log-maxsize": "10000"}
	kubeletArgs   = map[string]string{"alsologtostderr": "true", "container-runtime": "docker"}
)
func TestWorkloadClusterCreation(t *testing.T) {
	version := os.Getenv("KUBERNETES_VERSION")
	if version == "" {
		version = "1.17.5"
		fmt.Printf("Using default version: '%s'\n", version)
	}
	c := getTestContext(t)
	defer c.cleanup()
	machineInfo := createFootlooseMachines(c)
	defer deleteFootlooseMachines(c)
	setupNetworking(c)
	ensureAllManagementPodsAreRunning(c)
	setupProviderRepository(c)
	installProvider(c)
	c.ConfigureHAProxy("127.0.0.1", 2222)
	installNamespace(c)
	installMachinePool(c, machineInfo)
	ensureAllManagementPodsAreRunning(c)
	ensureSwapSettingsArePersisted(c)
	createWorkloadCluster(c, version)
	workloadKubeConfig := getWorkloadKubeconfig(c)
	ensureAllWorkloadNodesAreRunning(c, workloadKubeConfig)
	applyNewDockerConfigPlusAPIServerAndKubeletArguments(c, workloadKubeConfig)
	ensureAllWorkloadNodesStoppedRunning(c, workloadKubeConfig)
	ensureAllWorkloadNodesAreRunning(c, workloadKubeConfig)
	ensureNewArgumentsWereProcessed(c)
	ensureSwapShutdownPersists(c)
}
func installCertManager(c *testContext) {
	log.Info("Installing cert manager...")
	c.runAndCheckError("kubectl", "apply", "--validate=false", "-f", "https://github.com/jetstack/cert-manager/releases/download/v0.11.1/cert-manager.yaml")
}
func setupNetworking(c *testContext) {
	c.runAndCheckError("docker", "network", "connect", "bridge", "kind-control-plane")
}
func setupProviderRepository(c *testContext) {
	log.Info("Setting up provider repository...")
	providerRepositoryDir := filepath.Join(c.testDir, "..", "..", "..", "local-repository")
	c.runAndCheckError("cp", "-r", providerRepositoryDir, c.tmpDir)
	t, err := template.New("clusterctlconfig").Parse(clusterctlConfig)
	require.NoError(c.t, err)
	var populated bytes.Buffer
	err = t.Execute(&populated, struct {
		HomeDir string
	}{
		c.tmpDir,
	})
	require.NoError(c.t, err)
	configDir := filepath.Join(c.tmpDir, ".cluster-api")
	os.MkdirAll(configDir, 0777)
	ioutil.WriteFile(filepath.Join(configDir, "clusterctl.yaml"), populated.Bytes(), 0600)
}
func installProvider(c *testContext) {
	log.Info("Installing existinginfra provider...")
	c.runWithConfig(
		commandConfig{
			Env:        c.getProviderEnvironment(),
			Stdout:     os.Stdout,
			Stderr:     os.Stderr,
			CheckError: true},
		filepath.Join(c.tmpDir, "clusterctl"), "init", "--infrastructure=existinginfra")
}
func installNamespace(c *testContext) {
	log.Info("Installing namespace...")
	c.applyManagementManifest(testNamespace)
}
func installMachinePool(c *testContext, info []capeios.MachineInfo) {
	log.Info("Installing machine pool...")
	t, err := template.New("machine-pool").Parse(testPoolTemplate)
	require.NoError(c.t, err)
	infostr, err := json.Marshal(info)
	require.NoError(c.t, err)
	encoded := base64.StdEncoding.EncodeToString(infostr)
	var populated bytes.Buffer
	err = t.Execute(&populated, struct {
		SecretData string
	}{
		encoded,
	})
	require.NoError(c.t, err)
	c.applyManagementManifest(string(populated.Bytes()))
}
func applyNewDockerConfigPlusAPIServerAndKubeletArguments(c *testContext, kubeconfig string) {
	eic := getExistingInfraCluster(c)
	cleanJson := eic.Annotations["kubectl.kubernetes.io/last-applied-configuration"]
	var cleanEic v1alpha3.ExistingInfraCluster
	err := json.Unmarshal([]byte(cleanJson), &cleanEic)
	require.NoError(c.t, err)
	eic = &cleanEic
	files := eic.Spec.OS.Files
	var contentMap map[string]interface{}
	for fidx := range files {
		file := &files[fidx]
		if file.Destination == "/etc/docker/daemon.json" {
			err := json.Unmarshal([]byte(file.Source.Contents), &contentMap)
			require.NoError(c.t, err)
			logopts, ok := contentMap["log-opts"]
			require.True(c.t, ok)
			logopts.(map[string]interface{})["labels"] = "io.kubernetes.pod.namespace,io.kubernetes.pod.name,io.kubernetes.container.name"
			bytes, err := json.Marshal(logopts)
			require.NoError(c.t, err)
			file.Source.Contents = string(bytes)
			break
		}
	}
	aargs := []v1alpha3.ServerArgument{}
	for name, value := range apiServerArgs {
		aargs = append(aargs, v1alpha3.ServerArgument{Name: name, Value: value})
	}
	eic.Spec.APIServer.ExtraArguments = aargs
	kargs := []v1alpha3.ServerArgument{}
	for name, value := range kubeletArgs {
		kargs = append(kargs, v1alpha3.ServerArgument{Name: name, Value: value})
	}
	eic.Spec.KubeletArguments = kargs
	bytes, err := yaml.Marshal(eic)
	require.NoError(c.t, err)
	c.applyWorkloadManifest(string(bytes), kubeconfig)
}
func getExistingInfraCluster(c *testContext) *v1alpha3.ExistingInfraCluster {
	cmanifest, _, err := c.runCollectingOutput("kubectl", "get", "existinginfracluster", "test-cluster", "--namespace=test", "-o", "json")
	require.NoError(c.t, err)
	var eic v1alpha3.ExistingInfraCluster
	err = json.Unmarshal(cmanifest, &eic)
	require.NoError(c.t, err)
	return &eic
}
type conn struct {
	ip   string
	port string
}