func (d *DRPCInstance) RunRelocate() (bool, error) {
	d.log.Info("Entering RunRelocate", "state", d.getLastDRState())
	const done = true
	preferredCluster := d.instance.Spec.PreferredCluster
	preferredClusterNamespace := preferredCluster
	homeCluster, err := d.validateRelocation(preferredCluster)
	if err != nil {
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), err.Error())
		return !done, err
	}
	if d.hasAlreadySwitchedOver(preferredCluster) {
		err := d.EnsureCleanup(preferredCluster)
		if err != nil {
			return !done, err
		}
		return done, nil
	}
	if homeCluster != "" && homeCluster != preferredCluster && !d.readyToSwitchOver(homeCluster) {
		errMsg := fmt.Sprintf("current cluster (%s) has not completed protection actions", homeCluster)
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), errMsg)
		return !done, fmt.Errorf(errMsg)
	}
	return d.relocate(preferredCluster, preferredClusterNamespace, rmn.Relocating)
}
runRelocate checks if pre-conditions for relocation are met, and if so performs the relocation
Pre-requisites for relocation are checked as follows:
- The exists at least one VRG across clusters (there is no state where we do not have a VRG as
primary or secondary once initial deployment is complete)
- Ensures that there is only one primary, before further state transitions
- If there are multiple primaries, wait for one of the primaries to transition
to a secondary. This can happen if MCV reports older VRG state as MW is being applied
to the cluster.
- Check if peers are ready
- If there are secondaries in flight, ensure they report secondary as the observed state
before moving forward
- preferredCluster should not report as Secondary, as it will never transition out of delete state
in the future, as there would be no primary. This can happen, if in between relocate the
preferred cluster was switched
- User needs to recover by changing the preferredCluster back to the initial intent
- Check if we already relocated to the preferredCluster, and ensure cleanup actions
- Check if current primary (that is not the preferred cluster), is ready to switch over
- Relocate!
package controllers
import (
	"context"
	"fmt"
	"reflect"
	"time"
	"github.com/ghodss/yaml"
	"github.com/go-logr/logr"
	ocmworkv1 "github.com/open-cluster-management/api/work/v1"
	plrv1 "github.com/open-cluster-management/multicloud-operators-placementrule/pkg/apis/apps/v1"
	errorswrapper "github.com/pkg/errors"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	rmn "github.com/ramendr/ramen/api/v1alpha1"
	rmnutil "github.com/ramendr/ramen/controllers/util"
	"github.com/prometheus/client_golang/prometheus"
	"sigs.k8s.io/controller-runtime/pkg/metrics"
)
type DRPCInstance struct {
	reconciler           *DRPlacementControlReconciler
	ctx                  context.Context
	log                  logr.Logger
	instance             *rmn.DRPlacementControl
	drPolicy             *rmn.DRPolicy
	needStatusUpdate     bool
	mcvRequestInProgress bool
	userPlacementRule    *plrv1.PlacementRule
	drpcPlacementRule    *plrv1.PlacementRule
	vrgs                 map[string]*rmn.VolumeReplicationGroup
	mwu                  rmnutil.MWUtil
	metricsTimer         timerInstance
}
func (d *DRPCInstance) startProcessing() bool {
	d.log.Info("Starting to process placement")
	requeue := true
	done, processingErr := d.processPlacement()
	if d.shouldUpdateStatus() || d.statusUpdateTimeElapsed() {
		if err := d.reconciler.updateDRPCStatus(d.instance, d.userPlacementRule); err != nil {
			d.log.Error(err, "failed to update status")
			return requeue
		}
	}
	if processingErr != nil {
		d.log.Info("Process placement", "error", processingErr.Error())
		return requeue
	}
	requeue = !done
	d.log.Info("Completed processing placement", "requeue", requeue)
	return requeue
}
func (d *DRPCInstance) processPlacement() (bool, error) {
	d.log.Info("Process DRPC Placement", "DRAction", d.instance.Spec.Action)
	switch d.instance.Spec.Action {
	case rmn.ActionFailover:
		return d.RunFailover()
	case rmn.ActionRelocate:
		return d.RunRelocate()
	}
	return d.RunInitialDeployment()
}
func (d *DRPCInstance) RunInitialDeployment() (bool, error) {
	d.log.Info("Running initial deployment")
	const done = true
	homeCluster, homeClusterNamespace := d.getHomeCluster()
	if homeCluster == "" {
		err := fmt.Errorf("PreferredCluster not set and unable to find home cluster in DRPCPlacementRule (%v)",
			d.drpcPlacementRule)
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), err.Error())
		rmnutil.ReportIfNotPresent(d.reconciler.eventRecorder, d.instance, corev1.EventTypeWarning,
			rmnutil.EventReasonDeployFail, err.Error())
		return !done, err
	}
	d.log.Info(fmt.Sprintf("Using homeCluster %s for initial deployment, uPlRule Decision %+v",
		homeCluster, d.userPlacementRule.Status.Decisions))
	deployed, clusterName := d.isDeployed(homeCluster)
	if deployed && clusterName != homeCluster {
		return done, nil
	}
	if deployed && d.isUserPlRuleUpdated(homeCluster) {
		if d.getLastDRState() == rmn.DRState("") {
			d.instance.Status.PreferredDecision = d.userPlacementRule.Status.Decisions[0]
			d.setDRState(rmn.Deployed)
			d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
				d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), "Already deployed")
		}
		return done, nil
	}
	result, err := d.startDeploying(homeCluster, homeClusterNamespace)
	if err != nil {
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), err.Error())
		return !done, err
	}
	d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
		d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), "Initial deployment completed")
	return result, nil
}
func (d *DRPCInstance) getHomeCluster() (string, string) {
	homeCluster := ""
	homeClusterNamespace := ""
	if d.instance.Spec.PreferredCluster != "" {
		homeCluster = d.instance.Spec.PreferredCluster
		homeClusterNamespace = homeCluster
	}
	if homeCluster == "" && d.drpcPlacementRule != nil && len(d.drpcPlacementRule.Status.Decisions) != 0 {
		homeCluster = d.drpcPlacementRule.Status.Decisions[0].ClusterName
		homeClusterNamespace = d.drpcPlacementRule.Status.Decisions[0].ClusterNamespace
	}
	return homeCluster, homeClusterNamespace
}
func (d *DRPCInstance) isDeployed(homeCluster string) (bool, string) {
	if d.isVRGAlreadyDeployedOnTargetCluster(homeCluster) {
		d.log.Info(fmt.Sprintf("Already deployed on homeCluster %s. Last state: %s",
			homeCluster, d.getLastDRState()))
		return true, homeCluster
	}
	clusterName, found := d.isVRGAlreadyDeployedElsewhere(homeCluster)
	if found {
		errMsg := fmt.Sprintf("Failed to place deployment on cluster %s, as it is active on cluster %s",
			homeCluster, clusterName)
		d.log.Info(errMsg)
		return true, clusterName
	}
	return false, ""
}
func (d *DRPCInstance) isUserPlRuleUpdated(homeCluster string) bool {
	return len(d.userPlacementRule.Status.Decisions) > 0 &&
		d.userPlacementRule.Status.Decisions[0].ClusterName == homeCluster
}
func (d *DRPCInstance) isVRGAlreadyDeployedOnTargetCluster(targetCluster string) bool {
	d.log.Info(fmt.Sprintf("isAlreadyDeployedAndProtected? - %q", reflect.ValueOf(d.vrgs).MapKeys()))
	return d.getCachedVRG(targetCluster) != nil
}
func (d *DRPCInstance) getCachedVRG(clusterName string) *rmn.VolumeReplicationGroup {
	vrg, found := d.vrgs[clusterName]
	if !found {
		d.log.Info("VRG not found on cluster", "Name", clusterName)
		return nil
	}
	return vrg
}
func (d *DRPCInstance) isVRGAlreadyDeployedElsewhere(clusterToSkip string) (string, bool) {
	for clusterName := range d.vrgs {
		if clusterName == clusterToSkip {
			continue
		}
		return clusterName, true
	}
	return "", false
}
func (d *DRPCInstance) startDeploying(homeCluster, homeClusterNamespace string) (bool, error) {
	const done = true
	d.setDRState(rmn.Deploying)
	d.setMetricsTimerFromDRState(rmn.Deploying)
	err := d.createVRGManifestWork(homeCluster)
	if err != nil {
		return false, err
	}
	err = d.updateUserPlacementRule(homeCluster, homeClusterNamespace)
	if err != nil {
		rmnutil.ReportIfNotPresent(d.reconciler.eventRecorder, d.instance, corev1.EventTypeWarning,
			rmnutil.EventReasonDeployFail, err.Error())
		return !done, err
	}
	if len(d.userPlacementRule.Status.Decisions) > 0 {
		d.instance.Status.PreferredDecision = d.userPlacementRule.Status.Decisions[0]
	}
	d.advanceToNextDRState()
	d.log.Info(fmt.Sprintf("DRPC (%+v)", d.instance))
	d.setMetricsTimerFromDRState(rmn.Deployed)
	return done, nil
}
func (d *DRPCInstance) RunFailover() (bool, error) {
	d.log.Info("Entering RunFailover", "state", d.getLastDRState())
	const done = true
	if d.instance.Spec.FailoverCluster == "" {
		msg := "failover cluster not set. FailoverCluster is a mandatory field"
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), msg)
		return done, fmt.Errorf(msg)
	}
	failoverClusterVRG, ok := d.vrgs[d.instance.Spec.FailoverCluster]
	if !ok {
		return d.switchToFailoverCluster()
	}
	if d.isVRGSecondary(failoverClusterVRG) {
		msg := "failover cluster has not recovered from last placement action"
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), msg)
		return done, fmt.Errorf(msg)
	}
	if d.hasAlreadySwitchedOver(d.instance.Spec.FailoverCluster) {
		err := d.EnsureCleanup(d.instance.Spec.FailoverCluster)
		if err != nil {
			return !done, err
		}
		return done, nil
	}
	return d.switchToFailoverCluster()
}
func (d *DRPCInstance) switchToFailoverCluster() (bool, error) {
	const done = true
	d.setDRState(rmn.FailingOver)
	d.setMetricsTimerFromDRState(rmn.FailingOver)
	d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
		d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), "Starting failover")
	d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionPeerReady, d.instance.Generation,
		metav1.ConditionFalse, rmn.ReasonNotStarted,
		fmt.Sprintf("Started failover to cluster %q", d.instance.Spec.FailoverCluster))
	curHomeCluster := d.getCurrentHomeClusterName()
	if curHomeCluster == "" {
		msg := "Invalid Failover request. Current home cluster does not exists"
		d.log.Info(msg)
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), msg)
		err := fmt.Errorf("failover requested on invalid state %v", d.instance.Status)
		rmnutil.ReportIfNotPresent(d.reconciler.eventRecorder, d.instance, corev1.EventTypeWarning,
			rmnutil.EventReasonSwitchFailed, err.Error())
		return done, err
	}
	newHomeCluster := d.instance.Spec.FailoverCluster
	err := d.changeToSecondaryIfCurrentNotNewHomeCluster(curHomeCluster, newHomeCluster)
	if err != nil {
		return !done, err
	}
	const restorePVs = true
	err = d.executeRelocation(newHomeCluster, "", restorePVs)
	if err != nil {
		d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
			d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), err.Error())
		rmnutil.ReportIfNotPresent(d.reconciler.eventRecorder, d.instance, corev1.EventTypeWarning,
			rmnutil.EventReasonSwitchFailed, err.Error())
		return !done, err
	}
	d.advanceToNextDRState()
	d.setMetricsTimerFromDRState(rmn.FailedOver)
	d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
		d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), "Failover completed")
	d.log.Info("Failover completed", "state", d.getLastDRState())
	return !done, nil
}
func (d *DRPCInstance) getCurrentHomeClusterName() string {
	curHomeCluster := ""
	if len(d.userPlacementRule.Status.Decisions) > 0 {
		curHomeCluster = d.userPlacementRule.Status.Decisions[0].ClusterName
	}
	if curHomeCluster == "" {
		curHomeCluster = d.instance.Status.PreferredDecision.ClusterName
	}
	return curHomeCluster
}
func (d *DRPCInstance) changeToSecondaryIfCurrentNotNewHomeCluster(curHomeCluster, newHomeCluster string) error {
	if curHomeCluster != newHomeCluster {
		err := d.updateVRGStateToSecondary(curHomeCluster)
		if err != nil {
			msg := "Failed to update existing VRG manifestwork to secondary"
			d.log.Error(err, msg)
			d.setDRPCCondition(&d.instance.Status.Conditions, rmn.ConditionAvailable, d.instance.Generation,
				d.getConditionStatusForTypeAvailable(), string(d.instance.Status.Phase), msg)
			rmnutil.ReportIfNotPresent(d.reconciler.eventRecorder, d.instance, corev1.EventTypeWarning,
				rmnutil.EventReasonSwitchFailed, err.Error())
			return err
		}
	}
	return nil
}