func removeMasterTaint(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
	if node.IsWorker {
		removeMasterTaintCmd := fmt.Sprintf("sudo -E /bin/sh -c \"/usr/local/bin/kubectl taint nodes %s node-role.kubernetes.io/master=:NoSchedule-\"", node.Name)
		_, err := mgr.Runner.ExecuteCmd(removeMasterTaintCmd, 5, true)
		if err != nil {
			return errors.Wrap(errors.WithStack(err), "Failed to remove master taint")
		}
	}
	return nil
}
removeMasterTaint is used to remove taint when current node both in controlPlane and worker.
package kubernetes
import (
	"encoding/base64"
	"fmt"
	"github.com/kubesphere/kubekey/pkg/kubernetes/config/v1beta2"
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	kubekeyapiv1alpha1 "github.com/kubesphere/kubekey/apis/kubekey/v1alpha1"
	"github.com/kubesphere/kubekey/pkg/plugins/dns"
	"github.com/kubesphere/kubekey/pkg/util"
	"github.com/kubesphere/kubekey/pkg/util/manager"
	"github.com/pkg/errors"
)
func GetClusterStatus(mgr *manager.Manager, _ *kubekeyapiv1alpha1.HostCfg) error {
	output, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"[ -f /etc/kubernetes/admin.conf ] && echo 'Cluster already exists.' || echo 'Cluster will be created.'\"", 0, false)
	if strings.Contains(output, "Cluster will be created") {
		return nil
	} else {
		if err != nil {
			return errors.Wrap(errors.WithStack(err), "Failed to find /etc/kubernetes/admin.conf")
		}
		mgr.ClusterStatus.IsExist = true
		if output, err := mgr.Runner.ExecuteCmd("sudo cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep 'image:' | awk -F '[:]' '{print $(NF-0)}'", 0, false); err != nil {
			return errors.Wrap(errors.WithStack(err), "Failed to find current version")
		} else {
			if !strings.Contains(output, "No such file or directory") {
				mgr.ClusterStatus.Version = output
			}
		}
		if err := loadKubeConfig(mgr); err != nil {
			return err
		}
		if err := getClusterInfo(mgr); err != nil {
			return err
		}
		checkKubeadmConfig, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"cat /etc/kubernetes/kubeadm-config.yaml\"", 0, false)
		if err != nil {
			return errors.Wrap(errors.WithStack(err), "Failed to find /etc/kubernetes/kubeadm-config.yaml")
		}
		if (mgr.ClusterStatus.BootstrapToken == "" || mgr.ClusterStatus.CertificateKey == "") && (strings.Contains(checkKubeadmConfig, "InitConfiguration") || strings.Contains(checkKubeadmConfig, "ClusterConfiguration")) {
			if err := getJoinInfo(mgr); err != nil {
				return err
			}
		}
	}
	return nil
}
func InitKubernetesCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
	if mgr.Runner.Index == 0 && !mgr.ClusterStatus.IsExist {
		if err := generateKubeadmConfig(mgr, node, manager.IsInitCluster, "", ""); err != nil {
			return err
		}
		resetCmd := "/usr/local/bin/kubeadm reset -f"
		if mgr.ContainerRuntimeEndpoint != "" {
			resetCmd = resetCmd + " --cri-socket " + mgr.ContainerRuntimeEndpoint
		}
		for i := 0; i < 3; i++ {
			_, err2 := mgr.Runner.ExecuteCmd("sudo env PATH=$PATH:/sbin:/usr/sbin /bin/sh -c \"/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --ignore-preflight-errors=FileExisting-crictl\"", 0, true)
			if err2 != nil {
				if i == 2 {
					return errors.Wrap(errors.WithStack(err2), "Failed to init kubernetes cluster")
				}
				_, _ = mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo env PATH=$PATH:/sbin:/usr/sbin /bin/sh -c \"%s\"", resetCmd), 0, true)
			} else {
				break
			}
		}
		if err := GetKubeConfigForControlPlane(mgr); err != nil {
			return err
		}
		if err := removeMasterTaint(mgr, node); err != nil {
			return err
		}
		if err := addWorkerLabel(mgr, node); err != nil {
			return err
		}
		if err := dns.CreateClusterDns(mgr); err != nil {
			return err
		}
		if err := InstallK8sCertsRenew(mgr); err != nil {
			return err
		}
	}
	return nil
}
func JoinNodesToCluster(mgr *manager.Manager, node *kubekeyapiv1alpha1.HostCfg) error {
	if !manager.ExistNode(mgr, node) {
		if err := generateKubeadmConfig(mgr, node, false, mgr.ClusterStatus.BootstrapToken, mgr.ClusterStatus.CertificateKey); err != nil {
			return err
		}
		resetCmd := "/usr/local/bin/kubeadm reset -f"
		if mgr.ContainerRuntimeEndpoint != "" {
			resetCmd = resetCmd + " --cri-socket " + mgr.ContainerRuntimeEndpoint
		}
		for i := 0; i < 3; i++ {
			_, err := mgr.Runner.ExecuteCmd("sudo env PATH=$PATH:/sbin:/usr/sbin /bin/sh -c \"/usr/local/bin/kubeadm join --config=/etc/kubernetes/kubeadm-config.yaml\"", 0, true)
			if err != nil {
				if i == 2 {
					return errors.Wrap(errors.WithStack(err), "Failed to add master to cluster")
				}
				_, _ = mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo env PATH=$PATH:/sbin:/usr/sbin /bin/sh -c \"%s\"", resetCmd), 0, true)
			} else {
				break
			}
		}
		if node.IsMaster {
			if err := GetKubeConfigForControlPlane(mgr); err != nil {
				return err
			}
			if err := removeMasterTaint(mgr, node); err != nil {
				return err
			}
			if err := InstallK8sCertsRenew(mgr); err != nil {
				return err
			}
		}
		if node.IsWorker && !node.IsMaster {
			if err := getKubeConfigForWorker(mgr); err != nil {
				return err
			}
		}
		if err := addWorkerLabel(mgr, node); err != nil {
			return err
		}
	}
	return nil
}
func loadKubeConfig(mgr *manager.Manager) error {
	kubeCfgBase64Cmd := "cat /etc/kubernetes/admin.conf | base64 --wrap=0"
	kubeConfigStr, err1 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", kubeCfgBase64Cmd), 1, false)
	if err1 != nil {
		return errors.Wrap(errors.WithStack(err1), "Failed to get cluster kubeconfig")
	}
	kubeConfigPath := filepath.Join(mgr.WorkDir, fmt.Sprintf("config-%s", mgr.ObjName))
	kubeconfigStr, err := base64.StdEncoding.DecodeString(kubeConfigStr)
	if err != nil {
		return err
	}
	oldServer := fmt.Sprintf("server: https://%s:%d", mgr.Cluster.ControlPlaneEndpoint.Domain, mgr.Cluster.ControlPlaneEndpoint.Port)
	newServer := fmt.Sprintf("server: https://%s:%d", mgr.Cluster.ControlPlaneEndpoint.Address, mgr.Cluster.ControlPlaneEndpoint.Port)
	newKubeconfigStr := strings.Replace(string(kubeconfigStr), oldServer, newServer, -1)
	if err := ioutil.WriteFile(kubeConfigPath, []byte(newKubeconfigStr), 0644); err != nil {
		return err
	}
	mgr.ClusterStatus.Kubeconfig = newKubeconfigStr
	return nil
}
func getClusterInfo(mgr *manager.Manager) error {
	allNodesInfo := map[string]string{}
	clusterInfo, err := mgr.Runner.ExecuteCmd("sudo -E /bin/sh -c \"/usr/local/bin/kubectl --no-headers=true get nodes -o custom-columns=:metadata.name,:status.nodeInfo.kubeletVersion,:status.addresses\"", 5, false)
	if err != nil {
		return errors.Wrap(errors.WithStack(err), "Failed to get cluster info")
	}
	ipv4Regexp, err := regexp.Compile("[\\d]+\\.[\\d]+\\.[\\d]+\\.[\\d]+")
	if err != nil {
		return err
	}
	ipv6Regexp, err := regexp.Compile("[a-f0-9]{1,4}(:[a-f0-9]{1,4}){7}|[a-f0-9]{1,4}(:[a-f0-9]{1,4}){0,7}::[a-f0-9]{0,4}(:[a-f0-9]{1,4}){0,7}")
	if err != nil {
		return err
	}
	tmp := strings.Split(clusterInfo, "\r\n")
	if len(tmp) >= 1 {
		for i := 0; i < len(tmp); i++ {
			if ipv4 := ipv4Regexp.FindStringSubmatch(tmp[i]); len(ipv4) != 0 {
				allNodesInfo[ipv4[0]] = ipv4[0]
			}
			if ipv6 := ipv6Regexp.FindStringSubmatch(tmp[i]); len(ipv6) != 0 {
				allNodesInfo[ipv6[0]] = ipv6[0]
			}
			if len(strings.Fields(tmp[i])) > 3 {
				allNodesInfo[strings.Fields(tmp[i])[0]] = strings.Fields(tmp[i])[1]
			} else {
				allNodesInfo[strings.Fields(tmp[i])[0]] = ""
			}
		}
	}
	mgr.ClusterStatus.AllNodesInfo = allNodesInfo
	return nil
}
func getJoinInfo(mgr *manager.Manager) error {
	uploadCertsCmd := "/usr/local/bin/kubeadm init phase upload-certs --config=/etc/kubernetes/kubeadm-config.yaml --upload-certs"
	output, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", uploadCertsCmd), 5, false)
	if err != nil {
		return errors.Wrap(errors.WithStack(err), "Failed to upload kubeadm certs")
	}
	reg := regexp.MustCompile("[0-9|a-z]{64}")
	mgr.ClusterStatus.CertificateKey = reg.FindAllString(output, -1)[0]
	err1 := patchKubeadmSecret(mgr)
	if err1 != nil {
		return err1
	}
	tokenCreateMasterCmd := "/usr/local/bin/kubeadm token create"
	output, err2 := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", tokenCreateMasterCmd), 5, false)
	if err2 != nil {
		return errors.Wrap(errors.WithStack(err2), "Failed to create kubeadm token")
	}
	reg = regexp.MustCompile("[0-9|a-z]{6}.[0-9|a-z]{16}")
	mgr.ClusterStatus.BootstrapToken = reg.FindAllString(output, -1)[0]
	return nil
}
func getKubeConfigForWorker(mgr *manager.Manager) error {
	createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube"
	chownKubeConfig := "chown $(id -u):$(id -g) -R $HOME/.kube"
	if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", createConfigDirCmd), 1, false); err != nil {
		return errors.Wrap(errors.WithStack(err), "Failed to create kube dir")
	}
	syncKubeconfigForRootCmd := fmt.Sprintf("sudo -E /bin/sh -c \"echo %s | base64 -d > %s\"", base64.StdEncoding.EncodeToString([]byte(mgr.ClusterStatus.Kubeconfig)), "/root/.kube/config")
	syncKubeconfigForUserCmd := fmt.Sprintf("echo %s | base64 -d > %s && %s", base64.StdEncoding.EncodeToString([]byte(mgr.ClusterStatus.Kubeconfig)), "$HOME/.kube/config", chownKubeConfig)
	if _, err := mgr.Runner.ExecuteCmd(syncKubeconfigForRootCmd, 1, false); err != nil {
		return errors.Wrap(errors.WithStack(err), "Failed to sync kube config")
	}
	if _, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", syncKubeconfigForUserCmd), 1, false); err != nil {
		return errors.Wrap(errors.WithStack(err), "Failed to sync kube config")
	}
	return nil
}
func GetKubeConfigForControlPlane(mgr *manager.Manager) error {
	createConfigDirCmd := "mkdir -p /root/.kube && mkdir -p $HOME/.kube"
	getKubeConfigCmd := "cp -f /etc/kubernetes/admin.conf /root/.kube/config"
	getKubeConfigCmdUsr := "cp -f /etc/kubernetes/admin.conf $HOME/.kube/config"
	chownKubeConfig := "chown $(id -u):$(id -g) $HOME/.kube/config"
	cmd := strings.Join([]string{createConfigDirCmd, getKubeConfigCmd, getKubeConfigCmdUsr, chownKubeConfig}, " && ")
	_, err := mgr.Runner.ExecuteCmd(fmt.Sprintf("sudo -E /bin/sh -c \"%s\"", cmd), 2, false)
	if err != nil {
		return errors.Wrap(errors.WithStack(err), "Failed to init kubernetes cluster")
	}
	return nil
}