func (r *Resilience) CheckMemberFailure(ctx context.Context) error {
	status, lastVersion := r.context.GetStatus()
	updateStatusNeeded := false
	if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
		for _, m := range list {
			log := r.log.With().
				Str("id", m.ID).
				Str("role", group.AsRole()).
				Logger()
			switch m.Phase {
			case api.MemberPhaseNone, api.MemberPhasePending:
				continue
			case api.MemberPhaseUpgrading, api.MemberPhaseRotating, api.MemberPhaseCleanOut, api.MemberPhaseRotateStart:
				if len(status.Plan) == 0 {
					log.Error().Msgf("No plan but member is in phase %s - marking as failed", m.Phase)
					m.Phase = api.MemberPhaseFailed
					status.Members.Update(m, group)
					updateStatusNeeded = true
				}
			}
			if m.Conditions.IsTrue(api.ConditionTypeReady) {
				continue
			}
			if !m.Phase.IsFailed() {
				if m.IsNotReadySince(time.Now().Add(-notReadySinceGracePeriod)) {
					failureAcceptable, reason, err := r.isMemberFailureAcceptable(ctx, group, m)
					if err != nil {
						log.Warn().Err(err).Msg("Failed to check is member failure is acceptable")
					} else if failureAcceptable {
						log.Info().Msg("Member is not ready for long time, marking is failed")
						m.Phase = api.MemberPhaseFailed
						status.Members.Update(m, group)
						updateStatusNeeded = true
					} else {
						log.Warn().Msgf("Member is not ready for long time, but it is not safe to mark it a failed because: %s", reason)
					}
				}
			}
			if !m.Phase.IsFailed() {
				count := m.RecentTerminationsSince(time.Now().Add(-recentTerminationsSinceGracePeriod))
				if count >= recentTerminationThreshold {
					failureAcceptable, reason, err := r.isMemberFailureAcceptable(ctx, group, m)
					if err != nil {
						log.Warn().Err(err).Msg("Failed to check is member failure is acceptable")
					} else if failureAcceptable {
						log.Info().Msg("Member has terminated too often in recent history, marking is failed")
						m.Phase = api.MemberPhaseFailed
						status.Members.Update(m, group)
						updateStatusNeeded = true
					} else {
						log.Warn().Msgf("Member has terminated too often in recent history, but it is not safe to mark it a failed because: %s", reason)
					}
				}
			}
		}
		return nil
	}); err != nil {
		return errors.WithStack(err)
	}
	if updateStatusNeeded {
		if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil {
			return errors.WithStack(err)
		}
	}
	return nil
}
CheckMemberFailure performs a check for members that should be in failed state because:
- They are frequently restarted
- They cannot be scheduled for a long time (TODO)
package resilience
import (
	"context"
	"time"
	"github.com/arangodb/kube-arangodb/pkg/util/errors"
	"github.com/arangodb/go-driver/agency"
	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
)
const (
	recentTerminationsSinceGracePeriod = time.Minute * 10
	notReadySinceGracePeriod           = time.Minute * 5
	recentTerminationThreshold         = 5
)