func (r *RestartReconciler) genGrepNodeCmd() []string {
	return []string{
		"bash", "-c", fmt.Sprintf("grep --regexp='^node[0-9]' %s", paths.AdminToolsConf),
	}
}
genGrepNodeCmd returns the command to run to get the nodes from admintools.conf
This function only handles grepping compat21 nodes.
package controllers
import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"time"
	"github.com/go-logr/logr"
	vapi "github.com/vertica/vertica-kubernetes/api/v1beta1"
	"github.com/vertica/vertica-kubernetes/pkg/cmds"
	"github.com/vertica/vertica-kubernetes/pkg/events"
	"github.com/vertica/vertica-kubernetes/pkg/names"
	"github.com/vertica/vertica-kubernetes/pkg/paths"
	"github.com/vertica/vertica-kubernetes/pkg/status"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	ctrl "sigs.k8s.io/controller-runtime"
)
const (
	RequeueWaitTimeInSeconds = 10
	AdminToolsMapFile = "/opt/vertica/config/ipMap.txt"
	StateUp = "UP"
)
type verticaIPLookup map[string]string
type RestartReconciler struct {
	VRec    *VerticaDBReconciler
	Log     logr.Logger
	Vdb     *vapi.VerticaDB  
	PRunner cmds.PodRunner
	PFacts  *PodFacts
	ATPod   types.NamespacedName  
}
func MakeRestartReconciler(vdbrecon *VerticaDBReconciler, log logr.Logger,
	vdb *vapi.VerticaDB, prunner cmds.PodRunner, pfacts *PodFacts) ReconcileActor {
	return &RestartReconciler{VRec: vdbrecon, Log: log, Vdb: vdb, PRunner: prunner, PFacts: pfacts}
}
func (r *RestartReconciler) Reconcile(ctx context.Context, req *ctrl.Request) (ctrl.Result, error) {
	if !r.Vdb.Spec.AutoRestartVertica {
		err := status.UpdateCondition(ctx, r.VRec.Client, r.Vdb,
			vapi.VerticaDBCondition{Type: vapi.AutoRestartVertica, Status: corev1.ConditionFalse},
		)
		return ctrl.Result{}, err
	}
	err := status.UpdateCondition(ctx, r.VRec.Client, r.Vdb,
		vapi.VerticaDBCondition{Type: vapi.AutoRestartVertica, Status: corev1.ConditionTrue},
	)
	if err != nil {
		return ctrl.Result{}, err
	}
	if err := r.PFacts.Collect(ctx, r.Vdb); err != nil {
		return ctrl.Result{}, err
	}
	if r.PFacts.getUpNodeCount() == 0 && r.Vdb.Spec.InitPolicy != vapi.CommunalInitPolicyScheduleOnly {
		return r.reconcileCluster(ctx)
	}
	return r.reconcileNodes(ctx)
}
func (r *RestartReconciler) reconcileCluster(ctx context.Context) (ctrl.Result, error) {
	if r.PFacts.areAllPodsRunningAndZeroInstalled() {
		r.Log.Info("All pods are running and none of them have an installation.  Nothing to restart.")
		return ctrl.Result{}, nil
	}
	if r.PFacts.countRunningAndInstalled() == 0 {
		r.Log.Info("Waiting for pods to come online that may need a Vertica restart")
		return ctrl.Result{Requeue: true}, nil
	}
	if ok := r.setATPod(); !ok {
		r.Log.Info("No pod found to run admintools from. Requeue reconciliation.")
		return ctrl.Result{Requeue: true}, nil
	}
	if res, err := r.reipNodes(ctx, r.PFacts.findReIPPods(false)); err != nil || res.Requeue {
		return res, err
	}
	dbDoesNotExist := !r.PFacts.doesDBExist().IsTrue()
	if !dbDoesNotExist {
		if upNodes, err := r.anyUpNodesInClusterState(ctx); err != nil || upNodes {
			return ctrl.Result{Requeue: upNodes}, err
		}
	}
	downPods := r.PFacts.findRestartablePods()
	if err := r.killOldProcesses(ctx, downPods); err != nil {
		return ctrl.Result{}, err
	}
	if dbDoesNotExist {
		return ctrl.Result{}, nil
	}
	return r.restartCluster(ctx)
}
func (r *RestartReconciler) reconcileNodes(ctx context.Context) (ctrl.Result, error) {
	downPods := r.PFacts.findRestartablePods()
	if len(downPods) > 0 {
		if ok := r.setATPod(); !ok {
			r.Log.Info("No pod found to run admintools from. Requeue reconciliation.")
			return ctrl.Result{Requeue: true}, nil
		}
		if res, err := r.restartPods(ctx, downPods); res.Requeue || res.RequeueAfter > 0 || err != nil {
			return res, err
		}
	}
	if r.Vdb.Spec.InitPolicy == vapi.CommunalInitPolicyScheduleOnly {
		return ctrl.Result{Requeue: r.shouldRequeueIfPodsNotRunning()}, nil
	}
	reIPPods := r.PFacts.findReIPPods(true)
	if len(reIPPods) > 0 {
		if ok := r.setATPod(); !ok {
			r.Log.Info("No pod found to run admintools from. Requeue reconciliation.")
			return ctrl.Result{Requeue: true}, nil
		}
		if res, err := r.reipNodes(ctx, reIPPods); res.Requeue || err != nil {
			return res, err
		}
	}
	return ctrl.Result{Requeue: r.shouldRequeueIfPodsNotRunning()}, nil
}
func (r *RestartReconciler) restartPods(ctx context.Context, pods []*PodFact) (ctrl.Result, error) {
	downPods, err := r.removePodsWithClusterUpState(ctx, pods)
	if err != nil {
		return ctrl.Result{}, err
	}
	if len(downPods) == 0 {
		return ctrl.Result{Requeue: true}, nil
	}
	vnodeList := genRestartVNodeList(downPods)
	ipList := genRestartIPList(downPods)
	if err := r.killOldProcesses(ctx, downPods); err != nil {
		return ctrl.Result{}, err
	}
	debugDumpAdmintoolsConf(ctx, r.PRunner, r.ATPod)
	cmd := r.genRestartNodeCmd(vnodeList, ipList)
	if stdout, err := r.execRestartPods(ctx, downPods, cmd); err != nil {
		if strings.Contains(stdout, "All nodes in the input are not down, can't restart") {
			return ctrl.Result{Requeue: false, RequeueAfter: time.Second * RequeueWaitTimeInSeconds}, nil
		}
		return ctrl.Result{}, fmt.Errorf("failed to restart pod(s) %w", err)
	}
	debugDumpAdmintoolsConf(ctx, r.PRunner, r.ATPod)
	r.PFacts.Invalidate()
	return ctrl.Result{Requeue: len(pods) > len(downPods)}, nil
}
func (r *RestartReconciler) anyUpNodesInClusterState(ctx context.Context) (bool, error) {
	clusterState, err := r.fetchClusterNodeStatus(ctx)
	if err != nil {
		return false, err
	}
	for _, state := range clusterState {
		if state == StateUp {
			return true, nil
		}
	}
	return false, nil
}
func (r *RestartReconciler) removePodsWithClusterUpState(ctx context.Context, pods []*PodFact) ([]*PodFact, error) {
	clusterState, err := r.fetchClusterNodeStatus(ctx)
	if err != nil {
		return nil, err
	}
	i := 0
	for _, pod := range pods {
		state, ok := clusterState[pod.vnodeName]
		if !ok || state != StateUp {
			pods[i] = pod
			i++
		}
	}
	return pods[:i], nil
}
func (r *RestartReconciler) fetchClusterNodeStatus(ctx context.Context) (map[string]string, error) {
	cmd := []string{
		"-t", "list_allnodes",
	}
	stdout, _, err := r.PRunner.ExecAdmintools(ctx, r.ATPod, names.ServerContainer, cmd...)
	if err != nil {
		return nil, err
	}
	return r.parseClusterNodeStatus(stdout), nil
}
func (r *RestartReconciler) parseClusterNodeStatus(stdout string) map[string]string {
	stateMap := map[string]string{}
	lines := strings.Split(stdout, "\n")
	const ColHeaderCount = 2
	if len(lines) <= ColHeaderCount {
		return stateMap
	}
	for _, line := range lines[ColHeaderCount:] {
		cols := strings.Split(line, "|")
		const ListNodesColCount = 4
		if len(cols) < ListNodesColCount {
			continue
		}
		vnode := strings.Trim(cols[0], " ")
		state := strings.Trim(cols[2], " ")
		stateMap[vnode] = state
	}
	return stateMap
}
func (r *RestartReconciler) execRestartPods(ctx context.Context, downPods []*PodFact, cmd []string) (string, error) {
	podNames := make([]string, 0, len(downPods))
	for _, pods := range downPods {
		podNames = append(podNames, pods.name.Name)
	}
	r.VRec.EVRec.Eventf(r.Vdb, corev1.EventTypeNormal, events.NodeRestartStarted,
		"Calling 'admintools -t restart_node' to restart the following pods: %s", strings.Join(podNames, ", "))
	start := time.Now()
	stdout, _, err := r.PRunner.ExecAdmintools(ctx, r.ATPod, names.ServerContainer, cmd...)
	if err != nil {
		r.VRec.EVRec.Event(r.Vdb, corev1.EventTypeWarning, events.NodeRestartFailed,
			"Failed while calling 'admintools -t restart_node'")
		return stdout, err
	}
	r.VRec.EVRec.Eventf(r.Vdb, corev1.EventTypeNormal, events.NodeRestartSucceeded,
		"Successfully called 'admintools -t restart_node' and it took %s", time.Since(start))
	return stdout, nil
}
func (r *RestartReconciler) reipNodes(ctx context.Context, pods []*PodFact) (ctrl.Result, error) {
	var mapFileContents []string
	oldIPs, err := r.fetchOldIPsFromNode(ctx, r.ATPod)
	if err != nil {
		return ctrl.Result{}, err
	}
	mapFileContents, ipChanging, ok := r.genMapFile(oldIPs, pods)
	if !ok {
		r.Log.Info("Could not generate the map file contents from nodes.  Requeue reconciliation.")
		return ctrl.Result{Requeue: true}, nil
	}
	if !ipChanging {
		return ctrl.Result{}, nil
	}
	cmd := genMapFileUploadCmd(mapFileContents)
	if _, _, err := r.PRunner.ExecInPod(ctx, r.ATPod, names.ServerContainer, cmd...); err != nil {
		return ctrl.Result{}, err
	}
	debugDumpAdmintoolsConf(ctx, r.PRunner, r.ATPod)
	cmd = genReIPCommand()
	if _, _, err := r.PRunner.ExecAdmintools(ctx, r.ATPod, names.ServerContainer, cmd...); err != nil {
		r.VRec.EVRec.Event(r.Vdb, corev1.EventTypeWarning, events.ReipFailed,
			"Attempt to run 'admintools -t re_ip' failed")
		return ctrl.Result{}, err
	}
	debugDumpAdmintoolsConf(ctx, r.PRunner, r.ATPod)
	return ctrl.Result{}, nil
}
func (r *RestartReconciler) restartCluster(ctx context.Context) (ctrl.Result, error) {
	cmd := []string{
		"-t", "start_db",
		"--database=" + r.Vdb.Spec.DBName,
		"--noprompt",
	}
	if r.Vdb.Spec.IgnoreClusterLease {
		cmd = append(cmd, "--ignore-cluster-lease")
	}
	if r.Vdb.Spec.RestartTimeout != 0 {
		cmd = append(cmd, fmt.Sprintf("--timeout=%d", r.Vdb.Spec.RestartTimeout))
	}
	r.VRec.EVRec.Event(r.Vdb, corev1.EventTypeNormal, events.ClusterRestartStarted,
		"Calling 'admintools -t start_db' to restart the cluster")
	start := time.Now()
	_, _, err := r.PRunner.ExecAdmintools(ctx, r.ATPod, names.ServerContainer, cmd...)
	if err != nil {
		r.VRec.EVRec.Event(r.Vdb, corev1.EventTypeWarning, events.ClusterRestartFailed,
			"Failed while calling 'admintools -t start_db'")
		return ctrl.Result{}, err
	}
	r.VRec.EVRec.Eventf(r.Vdb, corev1.EventTypeNormal, events.ClusterRestartSucceeded,
		"Successfully called 'admintools -t start_db' and it took %s", time.Since(start))
	return ctrl.Result{}, err
}
func genRestartVNodeList(downPods []*PodFact) []string {
	hostList := []string{}
	for _, v := range downPods {
		hostList = append(hostList, v.vnodeName)
	}
	return hostList
}
func genRestartIPList(downPods []*PodFact) []string {
	ipList := []string{}
	for _, v := range downPods {
		ipList = append(ipList, v.podIP)
	}
	return ipList
}
func (r *RestartReconciler) killOldProcesses(ctx context.Context, pods []*PodFact) error {
	for _, pod := range pods {
		cmd := []string{
			"bash", "-c", "for pid in $(pgrep ^vertica$); do kill -n SIGKILL $pid; done",
		}
		if _, _, err := r.PRunner.ExecInPod(ctx, pod.name, names.ServerContainer, cmd...); err != nil {
			return err
		}
	}
	return nil
}
func (r *RestartReconciler) genRestartNodeCmd(vnodeList, ipList []string) []string {
	cmd := []string{
		"-t", "restart_node",
		"--database=" + r.Vdb.Spec.DBName,
		"--hosts=" + strings.Join(vnodeList, ","),
		"--new-host-ips=" + strings.Join(ipList, ","),
		"--noprompt",
	}
	if r.Vdb.Spec.RestartTimeout != 0 {
		cmd = append(cmd, fmt.Sprintf("--timeout=%d", r.Vdb.Spec.RestartTimeout))
	}
	return cmd
}
func parseNodesFromAdmintoolConf(nodeText string) verticaIPLookup {
	ips := make(verticaIPLookup)
	rs := `^(node\d{4}) = ([\d.:a-fA-F]+),`
	re := regexp.MustCompile(rs)
	for _, line := range strings.Split(nodeText, "\n") {
		match := re.FindAllStringSubmatch(line, 1)
		if len(match) > 0 && len(match[0]) >= 3 {
			ips[match[0][1]] = match[0][2]
		}
	}
	return ips
}
func (r *RestartReconciler) fetchOldIPsFromNode(ctx context.Context, atPod types.NamespacedName) (verticaIPLookup, error) {
	cmd := r.genGrepNodeCmd()
	stdout, _, err := r.PRunner.ExecInPod(ctx, atPod, names.ServerContainer, cmd...)
	if err != nil {
		return verticaIPLookup{}, err
	}
	return parseNodesFromAdmintoolConf(stdout), nil
}