func (r *Reconciler) addDebugRuleEvaluationStatusesToStatus(ctx reconcileRequestContext) error {
	debugStatuses, err := sdkutil.ConvertDebugRuleEvaluationStatusesFromSageMaker(ctx.TrainingJobDescription.DebugRuleEvaluationStatuses)
	if err != nil {
		return err
	}
	ctx.TrainingJob.Status.DebugRuleEvaluationStatuses = debugStatuses
	if err = r.Status().Update(ctx, ctx.TrainingJob); err != nil {
		return err
	}
	return nil
}
Add information regarding the debugging statuses to the status fields.
package trainingjob
import (
	"context"
	"fmt"
	"time"
	"github.com/pkg/errors"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/predicate"
	trainingjobv1 "github.com/aws/amazon-sagemaker-operator-for-k8s/api/v1/trainingjob"
	"github.com/aws/amazon-sagemaker-operator-for-k8s/controllers"
	"github.com/aws/amazon-sagemaker-operator-for-k8s/controllers/sdkutil"
	"github.com/aws/amazon-sagemaker-operator-for-k8s/controllers/sdkutil/clientwrapper"
	aws "github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/sagemaker"
	"github.com/go-logr/logr"
)
const (
	ReconcilingTrainingJobStatus = "ReconcilingTrainingJob"
)
const (
	MaxTrainingJobNameLength = 63
)
type Reconciler struct {
	client.Client
	Log                   logr.Logger
	PollInterval          time.Duration
	createSageMakerClient clientwrapper.SageMakerClientWrapperProvider
	awsConfigLoader       controllers.AWSConfigLoader
}
func NewTrainingJobReconciler(client client.Client, log logr.Logger, pollInterval time.Duration) *Reconciler {
	return &Reconciler{
		Client:       client,
		Log:          log,
		PollInterval: pollInterval,
		createSageMakerClient: func(cfg aws.Config) clientwrapper.SageMakerClientWrapper {
			sess := controllers.CreateNewAWSSessionFromConfig(cfg)
			return clientwrapper.NewSageMakerClientWrapper(sagemaker.New(sess))
		},
		awsConfigLoader: controllers.NewAWSConfigLoader(),
	}
}
func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
	ctx := reconcileRequestContext{
		Context:     context.Background(),
		Log:         r.Log.WithValues("trainingjob", req.NamespacedName),
		TrainingJob: new(trainingjobv1.TrainingJob),
	}
	ctx.Log.Info("Getting resource")
	if err := r.Get(ctx, req.NamespacedName, ctx.TrainingJob); err != nil {
		ctx.Log.Info("Unable to fetch TrainingJob job", "reason", err)
		if apierrs.IsNotFound(err) {
			return controllers.NoRequeue()
		}
		return controllers.RequeueImmediately()
	}
	if err := r.reconcileTrainingJob(ctx); err != nil {
		ctx.Log.Info("Got error while reconciling, will retry", "err", err)
		return controllers.RequeueImmediately()
	}
	for _, debugRuleJob := range ctx.TrainingJob.Status.DebugRuleEvaluationStatuses {
		if controllers.GetOrDefault(debugRuleJob.RuleEvaluationStatus, "") == string(sagemaker.RuleEvaluationStatusInProgress) || controllers.GetOrDefault(debugRuleJob.RuleEvaluationStatus, "") == string(sagemaker.RuleEvaluationStatusStopping) {
			return controllers.RequeueAfterInterval(r.PollInterval, nil)
		}
	}
	switch ctx.TrainingJob.Status.TrainingJobStatus {
	case string(sagemaker.TrainingJobStatusCompleted):
		fallthrough
	case string(sagemaker.TrainingJobStatusFailed):
		fallthrough
	case string(sagemaker.TrainingJobStatusStopped):
		return controllers.NoRequeue()
	default:
		return controllers.RequeueAfterInterval(r.PollInterval, nil)
	}
}
type reconcileRequestContext struct {
	context.Context
	Log             logr.Logger
	SageMakerClient clientwrapper.SageMakerClientWrapper
	TrainingJob *trainingjobv1.TrainingJob
	TrainingJobDescription *sagemaker.DescribeTrainingJobOutput
	TrainingJobName string
}
func (r *Reconciler) reconcileTrainingJob(ctx reconcileRequestContext) error {
	var err error
	if ctx.TrainingJob.Status.TrainingJobStatus == "" {
		if err = r.updateStatus(ctx, controllers.InitializingJobStatus, ""); err != nil {
			return err
		}
	}
	if err = r.initializeContext(&ctx); err != nil {
		return r.updateStatusAndReturnError(ctx, string(sagemaker.TrainingJobStatusFailed), "", errors.Wrap(err, "Unable to initialize operator"))
	}
	if !controllers.HasDeletionTimestamp(ctx.TrainingJob.ObjectMeta) {
		if !controllers.ContainsString(ctx.TrainingJob.ObjectMeta.GetFinalizers(), controllers.SageMakerResourceFinalizerName) {
			ctx.TrainingJob.ObjectMeta.Finalizers = append(ctx.TrainingJob.ObjectMeta.Finalizers, controllers.SageMakerResourceFinalizerName)
			if err := r.Update(ctx, ctx.TrainingJob); err != nil {
				return errors.Wrap(err, "Failed to add finalizer")
			}
			ctx.Log.Info("Finalizer added")
		}
	}
	if ctx.TrainingJobDescription, err = ctx.SageMakerClient.DescribeTrainingJob(ctx, ctx.TrainingJobName); err != nil {
		return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to describe SageMaker training job"))
	}
	if ctx.TrainingJobDescription == nil {
		if controllers.HasDeletionTimestamp(ctx.TrainingJob.ObjectMeta) {
			return r.removeFinalizer(ctx)
		}
		if err = r.createTrainingJob(ctx); err != nil {
			return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to create training job"))
		}
		if ctx.TrainingJobDescription, err = ctx.SageMakerClient.DescribeTrainingJob(ctx, ctx.TrainingJobName); err != nil {
			return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to describe SageMaker training job"))
		}
	}
	if len(ctx.TrainingJobDescription.DebugRuleEvaluationStatuses) != len(ctx.TrainingJob.Status.DebugRuleEvaluationStatuses) {
		if err = r.addDebugRuleEvaluationStatusesToStatus(ctx); err != nil {
			return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to add debug job statuses to status"))
		}
	} else {
		for i, debugJob := range ctx.TrainingJobDescription.DebugRuleEvaluationStatuses {
			if *debugJob.RuleEvaluationStatus != controllers.GetOrDefault(ctx.TrainingJob.Status.DebugRuleEvaluationStatuses[i].RuleEvaluationStatus, "") {
				if err = r.addDebugRuleEvaluationStatusesToStatus(ctx); err != nil {
					return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to add debug job statuses to status"))
				}
				break
			}
		}
	}
	switch *ctx.TrainingJobDescription.TrainingJobStatus {
	case sagemaker.TrainingJobStatusInProgress:
		if controllers.HasDeletionTimestamp(ctx.TrainingJob.ObjectMeta) {
			if _, err := ctx.SageMakerClient.StopTrainingJob(ctx, ctx.TrainingJobName); err != nil && !clientwrapper.IsStopTrainingJob404Error(err) {
				return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to delete training job"))
			}
			if ctx.TrainingJobDescription, err = ctx.SageMakerClient.DescribeTrainingJob(ctx, ctx.TrainingJobName); err != nil {
				return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to describe SageMaker training job"))
			}
		}
		break
	case sagemaker.TrainingJobStatusCompleted:
		if err = r.addModelPathToStatus(ctx); err != nil {
			return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", errors.Wrap(err, "Unable to add model path to status"))
		}
		fallthrough
	case sagemaker.TrainingJobStatusStopped, sagemaker.TrainingJobStatusFailed:
		if controllers.HasDeletionTimestamp(ctx.TrainingJob.ObjectMeta) {
			return r.removeFinalizer(ctx)
		}
		break
	case sagemaker.TrainingJobStatusStopping:
		break
	default:
		unknownStateError := errors.New(fmt.Sprintf("Unknown Training Job Status: %s", *ctx.TrainingJobDescription.TrainingJobStatus))
		return r.updateStatusAndReturnError(ctx, ReconcilingTrainingJobStatus, "", unknownStateError)
	}
	primaryStatus := *ctx.TrainingJobDescription.TrainingJobStatus
	secondaryStatus := *ctx.TrainingJobDescription.SecondaryStatus
	additional := controllers.GetOrDefault(ctx.TrainingJobDescription.FailureReason, "")
	if *ctx.TrainingJobDescription.TrainingJobStatus == string(sagemaker.TrainingJobStatusStopping) {
		secondaryStatus = ""
	}
	if err = r.updateStatusWithAdditional(ctx, primaryStatus, secondaryStatus, additional); err != nil {
		return err
	}
	return nil
}
func (r *Reconciler) initializeContext(ctx *reconcileRequestContext) error {
	if ctx.TrainingJob.Spec.TrainingJobName != nil && len(*ctx.TrainingJob.Spec.TrainingJobName) > 0 {
		ctx.TrainingJobName = *ctx.TrainingJob.Spec.TrainingJobName
	} else {
		ctx.TrainingJobName = controllers.GetGeneratedJobName(ctx.TrainingJob.ObjectMeta.GetUID(), ctx.TrainingJob.ObjectMeta.GetName(), MaxTrainingJobNameLength)
		ctx.TrainingJob.Spec.TrainingJobName = &ctx.TrainingJobName
		if err := r.Update(ctx, ctx.TrainingJob); err != nil {
			ctx.Log.Info("Error while updating training job name in spec")
			return err
		}
	}
	ctx.Log.Info("TrainingJob", "name", ctx.TrainingJobName)
	awsConfig, err := r.awsConfigLoader.LoadAWSConfigWithOverrides(ctx.TrainingJob.Spec.Region, ctx.TrainingJob.Spec.SageMakerEndpoint)
	if err != nil {
		ctx.Log.Error(err, "Error loading AWS config")
		return err
	}
	ctx.SageMakerClient = r.createSageMakerClient(awsConfig)
	ctx.Log.Info("Loaded AWS config")
	return nil
}
func (r *Reconciler) createTrainingJob(ctx reconcileRequestContext) error {
	var createTrainingJobInput sagemaker.CreateTrainingJobInput
	if ctx.TrainingJob.Spec.TrainingJobName == nil || len(*ctx.TrainingJob.Spec.TrainingJobName) == 0 {
		ctx.TrainingJob.Spec.TrainingJobName = &ctx.TrainingJobName
	}
	createTrainingJobInput = sdkutil.CreateCreateTrainingJobInputFromSpec(ctx.TrainingJob.Spec)
	ctx.Log.Info("Creating TrainingJob in SageMaker", "input", createTrainingJobInput)
	if _, err := ctx.SageMakerClient.CreateTrainingJob(ctx, &createTrainingJobInput); err != nil {
		return errors.Wrap(err, "Unable to create Training Job")
	}
	return nil
}
func (r *Reconciler) addModelPathToStatus(ctx reconcileRequestContext) error {
	var err error
	ctx.Log.Info("Training has completed updating model path")
	const outputPath string = "/output/model.tar.gz"
	ctx.TrainingJob.Status.ModelPath = *ctx.TrainingJob.Spec.OutputDataConfig.S3OutputPath + ctx.TrainingJob.Status.SageMakerTrainingJobName + outputPath
	if err = r.Status().Update(ctx, ctx.TrainingJob); err != nil {
		return err
	}
	return nil
}
func (r *Reconciler) removeFinalizer(ctx reconcileRequestContext) error {
	var err error
	ctx.TrainingJob.ObjectMeta.Finalizers = controllers.RemoveString(ctx.TrainingJob.ObjectMeta.Finalizers, controllers.SageMakerResourceFinalizerName)
	if err = r.Update(ctx, ctx.TrainingJob); err != nil {
		return errors.Wrap(err, "Failed to remove finalizer")
	}
	ctx.Log.Info("Finalizer has been removed")
	return nil
}