func (r *HostingDeploymentReconciler) updateStatusWithAdditional(ctx reconcileRequestContext, endpointStatus, additional string) error {
	ctx.Log.Info("updateStatusWithAdditional", "endpointStatus", endpointStatus, "additional", additional)
	deploymentStatus := &ctx.Deployment.Status
	endpoint := ctx.EndpointDescription
	deploymentStatus.Additional = additional
	deploymentStatus.EndpointStatus = endpointStatus
	deploymentStatus.LastCheckTime = Now()
	if endpoint != nil {
		deploymentStatus.EndpointName = GetOrDefault(endpoint.EndpointName, "")
		deploymentStatus.EndpointArn = GetOrDefault(endpoint.EndpointArn, "")
		deploymentStatus.FailureReason = GetOrDefault(endpoint.FailureReason, "")
		deploymentStatus.LastModifiedTime = convertTimeOrDefault(endpoint.LastModifiedTime, nil)
		deploymentStatus.CreationTime = convertTimeOrDefault(endpoint.CreationTime, nil)
		if converted, err := sdkutil.ConvertProductionVariantSummarySlice(endpoint.ProductionVariants); err != nil {
			return errors.Wrap(err, "Unable to interpret ProductionVariant for status")
		} else {
			deploymentStatus.ProductionVariants = converted
		}
	}
	deploymentStatus.ModelNames = sdkutil.ConvertMapToKeyValuePairSlice(ctx.ModelNames)
	deploymentStatus.EndpointConfigName = ctx.EndpointConfigName
	if ctx.Deployment.Spec.Region != nil && ctx.EndpointName != "" {
		deploymentStatus.EndpointUrl = "https://runtime.sagemaker." + *ctx.Deployment.Spec.Region + ".amazonaws.com/endpoints/" + ctx.EndpointName + "/invocations"
	} else {
		deploymentStatus.EndpointUrl = ""
	}
	if err := r.Status().Update(ctx, ctx.Deployment); err != nil {
		err = errors.Wrap(err, "Unable to update status")
		ctx.Log.Info("Error while updating status.", "err", err)
		return err
	}
	return nil
}
Update the status and other informational fields. The "additional" parameter should be used to convey additional error information. Leave empty to omit.
Returns an error if there was a failure to update.
package hosting
import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"
	"github.com/go-logr/logr"
	"github.com/pkg/errors"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	types "k8s.io/apimachinery/pkg/types"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/predicate"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/sagemaker"
	"github.com/aws/aws-sdk-go/service/sagemaker/sagemakeriface"
	hostingv1 "github.com/aws/amazon-sagemaker-operator-for-k8s/api/v1/hostingdeployment"
	. "github.com/aws/amazon-sagemaker-operator-for-k8s/controllers"
	"github.com/aws/amazon-sagemaker-operator-for-k8s/controllers/sdkutil"
	"github.com/aws/amazon-sagemaker-operator-for-k8s/controllers/sdkutil/clientwrapper"
)
const (
	ReconcilingEndpointStatus = "ReconcilingEndpoint"
	MaxResourceNameLength = 63
)
type HostingDeploymentReconciler struct {
	client.Client
	Log          logr.Logger
	PollInterval time.Duration
	awsConfigLoader                AWSConfigLoader
	createModelReconciler          ModelReconcilerProvider
	createEndpointConfigReconciler EndpointConfigReconcilerProvider
	createSageMakerClient          SageMakerClientProvider
}
func NewHostingDeploymentReconciler(client client.Client, log logr.Logger, pollInterval time.Duration) *HostingDeploymentReconciler {
	return &HostingDeploymentReconciler{
		Client:       client,
		Log:          log,
		PollInterval: pollInterval,
		createSageMakerClient: func(cfg aws.Config) sagemakeriface.SageMakerAPI {
			return sagemaker.New(CreateNewAWSSessionFromConfig(cfg))
		},
		awsConfigLoader:                NewAWSConfigLoader(),
		createModelReconciler:          NewModelReconciler,
		createEndpointConfigReconciler: NewEndpointConfigReconciler,
	}
}
func (r *HostingDeploymentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
	ctx := reconcileRequestContext{
		Context:    context.Background(),
		Log:        r.Log.WithValues("hostingdeployment", req.NamespacedName),
		Deployment: new(hostingv1.HostingDeployment),
	}
	if err := r.Get(ctx, req.NamespacedName, ctx.Deployment); err != nil {
		ctx.Log.Info("Unable to fetch HostingDeployment", "reason", err)
		if apierrs.IsNotFound(err) {
			return NoRequeue()
		} else {
			return RequeueImmediately()
		}
	}
	if err := r.reconcileHostingDeployment(ctx); err != nil {
		ctx.Log.Info("Got error while reconciling, will retry", "err", err)
		return RequeueImmediately()
	} else {
		return RequeueAfterInterval(r.PollInterval, nil)
	}
}
type reconcileRequestContext struct {
	context.Context
	Log                      logr.Logger
	SageMakerClient          clientwrapper.SageMakerClientWrapper
	ModelReconciler          ModelReconciler
	EndpointConfigReconciler EndpointConfigReconciler
	Deployment *hostingv1.HostingDeployment
	EndpointDescription *sagemaker.DescribeEndpointOutput
	EndpointName string
	ModelNames map[string]*string
	EndpointConfigName string
}
func (r *HostingDeploymentReconciler) reconcileHostingDeployment(ctx reconcileRequestContext) error {
	var err error
	if ctx.Deployment.Status.EndpointStatus == "" {
		if err = r.updateStatus(ctx, InitializingJobStatus); err != nil {
			return err
		}
	}
	if err = r.initializeContext(&ctx); err != nil {
		return r.updateStatusAndReturnError(ctx, string(sagemaker.EndpointStatusFailed), errors.Wrap(err, "Unable to initialize operator"))
	}
	if !HasDeletionTimestamp(ctx.Deployment.ObjectMeta) {
		if !ContainsString(ctx.Deployment.ObjectMeta.GetFinalizers(), SageMakerResourceFinalizerName) {
			ctx.Deployment.ObjectMeta.Finalizers = append(ctx.Deployment.ObjectMeta.Finalizers, SageMakerResourceFinalizerName)
			if err := r.Update(ctx, ctx.Deployment); err != nil {
				return errors.Wrap(err, "Failed to add finalizer")
			}
			ctx.Log.Info("Finalizer added")
		}
	}
	if ctx.EndpointConfigName, err = ctx.EndpointConfigReconciler.GetSageMakerEndpointConfigName(ctx, ctx.Deployment); err != nil {
		r.Log.Info("Unable to get EndpointConfigName", "err", err)
	}
	if ctx.ModelNames, err = ctx.ModelReconciler.GetSageMakerModelNames(ctx, ctx.Deployment); err != nil {
		r.Log.Info("Unable to get model names", "err", err)
	}
	if ctx.EndpointDescription, err = ctx.SageMakerClient.DescribeEndpoint(ctx, ctx.EndpointName); err != nil {
		return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to describe SageMaker endpoint"))
	}
	if ctx.EndpointDescription == nil {
		if HasDeletionTimestamp(ctx.Deployment.ObjectMeta) {
			return r.cleanupAndRemoveFinalizer(ctx)
		}
		if err = r.createEndpoint(ctx); err != nil {
			return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to create Endpoint"))
		}
		if ctx.EndpointDescription, err = ctx.SageMakerClient.DescribeEndpoint(ctx, ctx.EndpointName); err != nil {
			return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to describe SageMaker endpoint"))
		}
	}
	switch *ctx.EndpointDescription.EndpointStatus {
	case sagemaker.EndpointStatusInService:
		if !HasDeletionTimestamp(ctx.Deployment.ObjectMeta) {
			if err = r.handleUpdates(ctx); err != nil {
				return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to update SageMaker endpoint"))
			}
		}
		fallthrough
	case sagemaker.EndpointStatusFailed:
		if HasDeletionTimestamp(ctx.Deployment.ObjectMeta) {
			if _, err := ctx.SageMakerClient.DeleteEndpoint(ctx, &ctx.EndpointName); err != nil && !clientwrapper.IsDeleteEndpoint404Error(err) {
				return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to delete Endpoint"))
			}
		}
		break
	case sagemaker.EndpointStatusCreating:
		fallthrough
	case sagemaker.EndpointStatusDeleting:
		fallthrough
	case sagemaker.EndpointStatusOutOfService:
		fallthrough
	case sagemaker.EndpointStatusRollingBack:
		fallthrough
	case sagemaker.EndpointStatusSystemUpdating:
		fallthrough
	case sagemaker.EndpointStatusUpdating:
		r.Log.Info("Noop action, endpoint status does not allow modifications", "status", ctx.EndpointDescription.EndpointStatus)
	}
	status := *ctx.EndpointDescription.EndpointStatus
	if HasDeletionTimestamp(ctx.Deployment.ObjectMeta) && ctx.EndpointDescription.EndpointStatus != aws.String(sagemaker.EndpointStatusDeleting) {
		status = string(sagemaker.EndpointStatusDeleting)
	}
	if err = r.updateStatus(ctx, status); err != nil {
		return err
	}
	return nil
}
func (r *HostingDeploymentReconciler) handleUpdates(ctx reconcileRequestContext) error {
	var err error
	if err = r.reconcileEndpointResources(ctx, false); err != nil {
		return errors.Wrap(err, "Unable to reconcile endpoint resources")
	}
	if ctx.EndpointConfigName, err = ctx.EndpointConfigReconciler.GetSageMakerEndpointConfigName(ctx, ctx.Deployment); err != nil {
		return errors.Wrap(err, "Unable to get SageMaker EndpointConfigName for endpoint")
	}
	if ctx.EndpointConfigName == "" {
		return fmt.Errorf("Unable to get SageMaker EndpointConfigName for endpoint")
	}
	if *ctx.EndpointDescription.EndpointConfigName != ctx.EndpointConfigName {
		r.Log.Info("Endpoint needs update", "endpoint name", ctx.EndpointName, "actual config name", ctx.EndpointDescription.EndpointConfigName, "desired config name", ctx.EndpointConfigName)
		var output *sagemaker.UpdateEndpointOutput
		var excludeRetainedVariantProperties []*sagemaker.VariantProperty
		excludeRetainedVariantProperties = sdkutil.ConvertVariantPropertiesToSageMakerVariantProperties(ctx.Deployment.Spec.ExcludeRetainedVariantProperties)
		if output, err = ctx.SageMakerClient.UpdateEndpoint(ctx, ctx.EndpointName, ctx.EndpointConfigName, ctx.Deployment.Spec.RetainAllVariantProperties, excludeRetainedVariantProperties); err != nil && !clientwrapper.IsUpdateEndpoint404Error(err) {
			return errors.Wrap(err, "Unable to update Endpoint")
		}
		if output == nil {
			return fmt.Errorf("Unable to update Endpoint: Endpoint not found")
		}
	} else {
		r.Log.Info("Endpoint up to date")
		if err = r.reconcileEndpointResources(ctx, true); err != nil {
			return errors.Wrap(err, "Unable to reconcile endpoint resources")
		}
	}
	return nil
}
func (r *HostingDeploymentReconciler) createEndpoint(ctx reconcileRequestContext) error {
	var err error
	if err = r.reconcileEndpointResources(ctx, false); err != nil {
		return errors.Wrap(err, "Unable to reconcile Endpoint resources")
	}
	var createEndpointInput *sagemaker.CreateEndpointInput
	if createEndpointInput, err = r.createCreateEndpointInput(ctx); err != nil {
		return errors.Wrap(err, "Unable to create CreateEndpointInput")
	}
	r.Log.Info("Create endpoint", "input", createEndpointInput)
	if _, err := ctx.SageMakerClient.CreateEndpoint(ctx, createEndpointInput); err != nil {
		return errors.Wrap(err, "Unable to create Endpoint")
	}
	return nil
}
func (r *HostingDeploymentReconciler) cleanupAndRemoveFinalizer(ctx reconcileRequestContext) error {
	var err error
	if err = r.reconcileEndpointResources(ctx, true); err != nil {
		return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to clean up HostingDeployment"))
	}
	if !ctx.Deployment.ObjectMeta.GetDeletionTimestamp().IsZero() {
		ctx.Deployment.ObjectMeta.Finalizers = RemoveString(ctx.Deployment.ObjectMeta.Finalizers, SageMakerResourceFinalizerName)
		if err = r.Update(ctx, ctx.Deployment); err != nil {
			return errors.Wrap(err, "Failed to remove finalizer")
		}
		ctx.Log.Info("Finalizer has been removed")
	}
	return nil
}
func (r *HostingDeploymentReconciler) initializeContext(ctx *reconcileRequestContext) error {
	if ctx.Deployment.Spec.EndpointName != nil && len(*ctx.Deployment.Spec.EndpointName) > 0 {
		ctx.EndpointName = *ctx.Deployment.Spec.EndpointName
	} else {
		ctx.EndpointName = GetGeneratedJobName(ctx.Deployment.ObjectMeta.GetUID(), ctx.Deployment.ObjectMeta.GetName(), MaxResourceNameLength)
	}
	r.Log.Info("SageMaker EndpointName", "name", ctx.EndpointName)
	awsConfig, err := r.awsConfigLoader.LoadAWSConfigWithOverrides(ctx.Deployment.Spec.Region, ctx.Deployment.Spec.SageMakerEndpoint)
	if err != nil {
		ctx.Log.Error(err, "Error loading AWS config")
		return err
	}
	ctx.SageMakerClient = clientwrapper.NewSageMakerClientWrapper(r.createSageMakerClient(awsConfig))
	ctx.Log.Info("Loaded AWS config")
	ctx.ModelReconciler = r.createModelReconciler(r, ctx.Log)
	ctx.EndpointConfigReconciler = r.createEndpointConfigReconciler(r, ctx.Log)
	return nil
}
func (r *HostingDeploymentReconciler) reconcileEndpointResources(ctx reconcileRequestContext, shouldDeleteUnusedResources bool) error {
	r.Log.Info("Reconcile endpoint resources", "shouldDeleteUnusedResources", shouldDeleteUnusedResources)
	if err := ctx.ModelReconciler.Reconcile(ctx, ctx.Deployment, shouldDeleteUnusedResources); err != nil {
		return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to reconcile models"))
	}
	if err := ctx.EndpointConfigReconciler.Reconcile(ctx, ctx.Deployment, shouldDeleteUnusedResources); err != nil {
		return r.updateStatusAndReturnError(ctx, ReconcilingEndpointStatus, errors.Wrap(err, "Unable to reconcile EndpointConfigs"))
	}
	return nil
}
func (r *HostingDeploymentReconciler) createCreateEndpointInput(ctx reconcileRequestContext) (*sagemaker.CreateEndpointInput, error) {
	r.Log.Info("Create CreateEndpointInput")
	var endpointConfigName string
	var err error
	if endpointConfigName, err = ctx.EndpointConfigReconciler.GetSageMakerEndpointConfigName(ctx, ctx.Deployment); err != nil {
		return nil, err
	}
	createInput := &sagemaker.CreateEndpointInput{
		EndpointConfigName: &endpointConfigName,
		EndpointName:       &ctx.EndpointName,
		Tags:               sdkutil.ConvertTagSliceToSageMakerTagSlice(ctx.Deployment.Spec.Tags),
	}
	return createInput, nil
}
func (r *HostingDeploymentReconciler) updateStatusAndReturnError(ctx reconcileRequestContext, status string, reconcileErr error) error {
	if err := r.updateStatusWithAdditional(ctx, status, reconcileErr.Error()); err != nil {
		return errors.Wrapf(reconcileErr, "Unable to update status with error. Status failure was caused by: '%s'", err.Error())
	}
	return reconcileErr
}
func (r *HostingDeploymentReconciler) updateStatus(ctx reconcileRequestContext, endpointStatus string) error {
	return r.updateStatusWithAdditional(ctx, endpointStatus, "")
}