func checkRestoreDone(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) bool {
	if cr.Status.RestoreProgress.AwaitingApp ||
		cr.Status.RestoreProgress.AwaitingStatus ||
		cr.Status.RestoreProgress.AwaitingResources {
		return false
	}
	patchedCR := *cr
	patchedCR.Labels = make(map[string]string)
	for key, value := range cr.Labels {
		if key != shared.RestoringLabel {
			patchedCR.Labels[key] = value
		}
	}
	patchErr := shared.Patch(
		context.TODO(),
		cr,
		&patchedCR,
	)
	if patchErr == nil {
		shared.LogInfo(
			reqLogger,
			cr,
			shared.EventReasonCluster,
			"resuming reconciliation",
		)
		return true
	}
	shared.LogInfof(
		reqLogger,
		cr,
		shared.EventReasonCluster,
		"failed to resume reconciliation: %s",
		patchErr.Error(),
	)
	cr.Status.RestoreProgress.Error = patchErr.Error()
	return false
}
checkRestoreDone will try to clear the being-restored label if all the
"waiting" flags are false. If that fails it will populate the error field
of the restore progress object. Return true if and only if we have
successfully cleared that label.
package kubedirectorcluster
import (
	"context"
	"time"
	kdv1 "github.com/bluek8s/kubedirector/pkg/apis/kubedirector/v1beta1"
	"github.com/bluek8s/kubedirector/pkg/catalog"
	"github.com/bluek8s/kubedirector/pkg/executor"
	"github.com/bluek8s/kubedirector/pkg/observer"
	"github.com/bluek8s/kubedirector/pkg/shared"
	"github.com/go-logr/logr"
	"github.com/google/uuid"
	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/errors"
)
func (r *ReconcileKubeDirectorCluster) handleRestore(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) error {
	oldStatus := cr.Status.DeepCopy()
	if cr.Status == nil {
		cr.Status = &kdv1.KubeDirectorClusterStatus{}
		cr.Status.Roles = make([]kdv1.RoleStatus, 0)
	}
	defer func() {
		statusChanged := false
		if cr.DeletionTimestamp == nil {
			statusChanged = !equality.Semantic.DeepEqual(cr.Status, oldStatus)
		}
		if !statusChanged {
			return
		}
		wait := time.Second
		maxWait := 4096 * time.Second
		for {
			cr.Status.GenerationUID = uuid.New().String()
			ClusterStatusGens.WriteStatusGen(cr.UID, cr.Status.GenerationUID)
			updateErr := executor.UpdateClusterStatus(cr, false, nil)
			if updateErr == nil {
				return
			}
			currentCluster, currentClusterErr := observer.GetCluster(
				cr.Namespace,
				cr.Name,
			)
			if currentClusterErr != nil {
				if errors.IsNotFound(currentClusterErr) {
					return
				}
			} else {
				if currentCluster.DeletionTimestamp != nil {
					return
				}
				if errors.IsConflict(updateErr) {
					currentCluster.Status = cr.Status
					*cr = *currentCluster
					continue
				}
			}
			if wait < maxWait {
				wait = wait * 2
			}
			shared.LogErrorf(
				reqLogger,
				updateErr,
				cr,
				shared.EventReasonCluster,
				"trying status update again in %v; failed",
				wait,
			)
			time.Sleep(wait)
		}
	}()
	if cr.Status.RestoreProgress == nil {
		shared.LogInfo(
			reqLogger,
			cr,
			shared.EventReasonCluster,
			"being restored: setting initial restore-progress flags",
		)
		cr.Status.RestoreProgress = &kdv1.RestoreProgress{
			AwaitingApp:       true,
			AwaitingStatus:    true,
			AwaitingResources: true,
		}
	}
	if checkRestoreDone(reqLogger, cr) {
		return nil
	}
	checkAppRestored(reqLogger, cr)
	checkStatusRestored(reqLogger, cr)
	if cr.Status.RestoreProgress.AwaitingStatus == false {
		checkResourcesRestored(reqLogger, cr)
	}
	return nil
}
func checkAppRestored(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) {
	_, err := catalog.FindApp(cr)
	catalogStr := "auto"
	if cr.Spec.AppCatalog != nil {
		catalogStr = *(cr.Spec.AppCatalog)
	}
	if err == nil {
		if cr.Status.RestoreProgress.AwaitingApp {
			cr.Status.RestoreProgress.AwaitingApp = false
			shared.LogInfof(
				reqLogger,
				cr,
				shared.EventReasonCluster,
				"being restored: kdapp %s (%s catalog) is present",
				cr.Spec.AppID,
				catalogStr,
			)
		}
	} else {
		cr.Status.RestoreProgress.AwaitingApp = true
		shared.LogInfof(
			reqLogger,
			cr,
			shared.EventReasonCluster,
			"being restored: awaiting kdapp %s (%s catalog)",
			cr.Spec.AppID,
			catalogStr,
		)
	}
}
func checkStatusRestored(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) {
	if cr.Status.RestoreProgress.AwaitingStatus == false {
		return
	}
	if cr.Annotations != nil {
		if annVal, ok := cr.Annotations[shared.StatusBackupAnnotation]; ok {
			if annVal == "false" {
				shared.LogInfo(
					reqLogger,
					cr,
					shared.EventReasonCluster,
					"being restored: kdstatusbackup was not ever created",
				)
				cr.Status.RestoreProgress.Error =
					"KD status backups were not enabled at the time this kdcluster " +
						"was snapshotted; it is not possible to do automatic status " +
						"recovery and resume reconciliation."
			}
		}
	}
	statusBackup, err := observer.GetStatusBackup(
		cr.Namespace,
		cr.Name,
	)
	if err == nil {
		if cr.Status.RestoreProgress.AwaitingStatus {
			cr.Status.RestoreProgress.AwaitingStatus = false
			shared.LogInfo(
				reqLogger,
				cr,
				shared.EventReasonCluster,
				"being restored: kdstatusbackup is present",
			)
			statusBackup.Spec.StatusBackup.RestoreProgress = cr.Status.RestoreProgress
			cr.Status = statusBackup.Spec.StatusBackup
		}
	} else {
		cr.Status.RestoreProgress.AwaitingStatus = true
		shared.LogInfo(
			reqLogger,
			cr,
			shared.EventReasonCluster,
			"being restored: awaiting kdstatusbackup",
		)
	}
}
func checkResourcesRestored(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) {
	resourcesPresent := clusterServiceExists(reqLogger, cr)
	resourcesPresent = resourcesPresent && roleResourcesExist(reqLogger, cr)
	if !shared.GetAllowRestoreWithoutConnections() {
		resourcesPresent = resourcesPresent && connectedResourcesExist(reqLogger, cr)
	}
	if resourcesPresent {
		if cr.Status.RestoreProgress.AwaitingResources {
			cr.Status.RestoreProgress.AwaitingResources = false
			shared.LogInfo(
				reqLogger,
				cr,
				shared.EventReasonCluster,
				"being restored: component resources are present",
			)
		}
	} else {
		cr.Status.RestoreProgress.AwaitingResources = true
		shared.LogInfo(
			reqLogger,
			cr,
			shared.EventReasonCluster,
			"being restored: awaiting component resources",
		)
	}
}
func clusterServiceExists(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) bool {
	clusterService := cr.Status.ClusterService
	if clusterService != "" {
		_, serviceErr := observer.GetService(
			cr.Namespace,
			clusterService,
		)
		if serviceErr != nil {
			shared.LogInfof(
				reqLogger,
				cr,
				shared.EventReasonCluster,
				"being restored: cluster service %s does not exist",
				clusterService,
			)
			return false
		}
	}
	return true
}
func roleResourcesExist(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) bool {
	for _, roleStatus := range cr.Status.Roles {
		if roleStatus.StatefulSet != "" {
			_, statefulSetErr := observer.GetStatefulSet(
				cr.Namespace,
				roleStatus.StatefulSet,
			)
			if statefulSetErr != nil {
				shared.LogInfof(
					reqLogger,
					cr,
					shared.EventReasonCluster,
					"being restored: statefulset %s does not exist",
					roleStatus.StatefulSet,
				)
				return false
			}
		}
		for _, memberStatus := range roleStatus.Members {
			memberService := memberStatus.Service
			if memberService != "" && memberService != zeroPortsService {
				_, serviceErr := observer.GetService(
					cr.Namespace,
					memberService,
				)
				if serviceErr != nil {
					shared.LogInfof(
						reqLogger,
						cr,
						shared.EventReasonCluster,
						"being restored: member service %s does not exist",
						memberService,
					)
					return false
				}
			}
		}
	}
	return true
}
func connectedResourcesExist(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
) bool {
	for _, kdcName := range cr.Spec.Connections.Clusters {
		_, kdcErr := observer.GetCluster(
			cr.Namespace,
			kdcName,
		)
		if kdcErr != nil {
			shared.LogInfof(
				reqLogger,
				cr,
				shared.EventReasonCluster,
				"being restored: connected kdcluster %s does not exist",
				kdcName,
			)
			return false
		}
	}
	for _, cfgName := range cr.Spec.Connections.ConfigMaps {
		_, cfgErr := observer.GetConfigMap(
			cr.Namespace,
			cfgName,
		)
		if cfgErr != nil {
			shared.LogInfof(
				reqLogger,
				cr,
				shared.EventReasonCluster,
				"being restored: connected ConfigMap %s does not exist",
				cfgName,
			)
			return false
		}
	}
	for _, secretName := range cr.Spec.Connections.Secrets {
		_, secretErr := observer.GetSecret(
			cr.Namespace,
			secretName,
		)
		if secretErr != nil {
			shared.LogInfof(
				reqLogger,
				cr,
				shared.EventReasonCluster,
				"being restored: connected Secret %s does not exist",
				secretName,
			)
			return false
		}
	}
	return true
}