func (ext *Checker) doFinalUpdateCheck(lastReportTime time.Time) (bool, error) {
	ext.log("witnessed checker pod removal. aborting watch for pod status report. check run skipped gracefully")
	currentUpdateTime, err := ext.getCheckLastUpdateTime()
	if err != nil {
		return false, err
	}
	ext.log("Last report time was:", lastReportTime, "vs", currentUpdateTime)
	if !currentUpdateTime.After(lastReportTime) {
		return false, nil
	}
	return true, nil
}
doFinalUpdateCheck is used to do one final update check before we conclude that the pod disappeared expectedly.
package external
import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	"github.com/google/uuid"
	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	apiv1 "k8s.io/api/core/v1"
	policyv1 "k8s.io/api/policy/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
	khcheckv1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khcheck/v1"
	khjobv1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khjob/v1"
	khstatev1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khstate/v1"
	"github.com/kuberhealthy/kuberhealthy/v2/pkg/checks/external/util"
)
const KHReportingURL = "KH_REPORTING_URL"
const KHRunUUID = "KH_RUN_UUID"
const KHDeadline = "KH_CHECK_RUN_DEADLINE"
const KHCheckNameAnnotationKey = "comcast.github.io/check-name"
const KHPodNamespace = "KH_POD_NAMESPACE"
const DefaultKuberhealthyReportingURL = "http://kuberhealthy.kuberhealthy.svc.cluster.local/externalCheckStatus"
const kuberhealthyRunIDLabel = "kuberhealthy-run-id"
const kuberhealthyCheckNameLabel = "kuberhealthy-check-name"
const defaultTimeout = time.Minute * 15
const defaultShutdownGracePeriod = time.Minute
var ErrPodRemovedExpectedly = errors.New("pod deleted expectedly")
var ErrPodRemovedUnexpectedly = errors.New("pod deleted unexpectedly")
var ErrPodDeletedBeforeRunning = errors.New("the khcheck check pod is deleted, waiting for start failed")
var DefaultName = "external-check"
var kubeConfigFile = filepath.Join(os.Getenv("HOME"), ".kube", "config")
var kuberhealthyNamespace = "kuberhealthy"
type Checker struct {
	CheckName                string  
	Namespace                string
	RunInterval              time.Duration  
	RunTimeout               time.Duration  
	KubeClient               *kubernetes.Clientset
	KHJobClient              *khjobv1.KHJobV1Client
	KHCheckClient            *khcheckv1.KHCheckV1Client
	KHStateClient            *khstatev1.KHStateV1Client
	PodSpec                  apiv1.PodSpec  
	OriginalPodSpec          apiv1.PodSpec  
	RunID                    string         
	KuberhealthyReportingURL string         
	ExtraAnnotations         map[string]string
	ExtraLabels              map[string]string
	Node                     string              
	currentCheckUUID         string              
	Debug                    bool                
	shutdownCTXFunc          context.CancelFunc  
	shutdownCTX              context.Context     
	wg                       sync.WaitGroup      
	hostname                 string              
	checkPodName             string              
	KHWorkload               khstatev1.KHWorkload
}
func init() {
	kuberhealthyNamespace = util.GetInstanceNamespace(kuberhealthyNamespace)
	log.Infoln("Kuberhealthy is located in the", kuberhealthyNamespace, "namespace.")
}
func New(client *kubernetes.Clientset, checkConfig *khcheckv1.KuberhealthyCheck, khCheckClient *khcheckv1.KHCheckV1Client, khStateClient *khstatev1.KHStateV1Client, reportingURL string) *Checker {
	return NewCheck(client, checkConfig, khCheckClient, khStateClient, reportingURL)
}
func NewCheck(client *kubernetes.Clientset, checkConfig *khcheckv1.KuberhealthyCheck, khCheckClient *khcheckv1.KHCheckV1Client, khStateClient *khstatev1.KHStateV1Client, reportingURL string) *Checker {
	if len(checkConfig.Namespace) == 0 {
		checkConfig.Namespace = "kuberhealthy"
	}
	log.Debugf("Creating external check from check config: %+v \n", checkConfig)
	return &Checker{
		Namespace:                checkConfig.Namespace,
		KHCheckClient:            khCheckClient,
		KHStateClient:            khStateClient,
		CheckName:                checkConfig.Name,
		KuberhealthyReportingURL: reportingURL,
		RunTimeout:               defaultTimeout,
		ExtraAnnotations:         make(map[string]string),
		ExtraLabels:              make(map[string]string),
		OriginalPodSpec:          checkConfig.Spec.PodSpec,
		PodSpec:                  checkConfig.Spec.PodSpec,
		KubeClient:               client,
		KHWorkload:               khstatev1.KHCheck,
	}
}
func NewJob(client *kubernetes.Clientset, jobConfig *khjobv1.KuberhealthyJob, khJobClient *khjobv1.KHJobV1Client, khStateClient *khstatev1.KHStateV1Client, reportingURL string) *Checker {
	if len(jobConfig.Namespace) == 0 {
		jobConfig.Namespace = "kuberhealthy"
	}
	log.Debugf("Creating kuberhealthy job from job config: %+v \n", jobConfig)
	return &Checker{
		Namespace:                jobConfig.Namespace,
		KHJobClient:              khJobClient,
		KHStateClient:            khStateClient,
		CheckName:                jobConfig.Name,
		KuberhealthyReportingURL: reportingURL,
		ExtraAnnotations:         make(map[string]string),
		ExtraLabels:              make(map[string]string),
		OriginalPodSpec:          jobConfig.Spec.PodSpec,
		PodSpec:                  jobConfig.Spec.PodSpec,
		KubeClient:               client,
		KHWorkload:               khstatev1.KHJob,
	}
}
func (ext *Checker) regeneratePodName() {
	var err error
	if len(ext.hostname) == 0 {
		ext.hostname, err = os.Hostname()
		if err != nil {
			log.Fatalln("Could not determine my hostname with error:", err)
		}
	}
	timeString := strconv.FormatInt(time.Now().Unix(), 10)
	ext.checkPodName = strings.ToLower(ext.CheckName + "-" + timeString)
}
func (ext *Checker) podName() string {
	return ext.checkPodName
}
func (ext *Checker) CurrentStatus() (bool, []string) {
	state, err := ext.getKHState()
	if err != nil {
		if k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found") {
			return true, []string{}
		}
		return false, []string{err.Error()}  
	}
	ext.log("length of error message slice:", len(state.Spec.Errors), state.Spec.Errors)
	if len(state.Spec.Errors) > 0 {
		ext.log("reporting check as OK=FALSE due to error messages > 0")
		return false, state.Spec.Errors
	}
	ext.log("reporting OK=TRUE due to error messages NOT > 0")
	return true, state.Spec.Errors
}
func (ext *Checker) Name() string {
	return ext.CheckName
}
func (ext *Checker) CheckNamespace() string {
	return ext.Namespace
}
func (ext *Checker) Interval() time.Duration {
	return ext.RunInterval
}
func (ext *Checker) Timeout() time.Duration {
	return ext.RunTimeout
}
func (ext *Checker) Run(ctx context.Context, client *kubernetes.Clientset) error {
	ext.KubeClient = client
	err := ext.setNewCheckUUID()
	if err != nil {
		return err
	}
	ext.log("Running external check iteration")
	err = ext.RunOnce(ctx)
	if err != nil && err.Error() == ErrPodRemovedExpectedly.Error() {
		ext.log("pod was removed during check expectedly. skipping this run")
		return ErrPodRemovedExpectedly
	}
	if err != nil {
		ext.log("Error with running external check:", err)
		return err
	}
	return nil
}
func (ext *Checker) getCheck() (*khcheckv1.KuberhealthyCheck, error) {
	log.Debugln("Fetching check", ext.CheckName, "in namespace", ext.Namespace)
	checkConfig, err := ext.KHCheckClient.KuberhealthyChecks(ext.Namespace).Get(ext.CheckName, metav1.GetOptions{})
	if err != nil {
		return &khcheckv1.KuberhealthyCheck{}, err
	}
	return &checkConfig, err
}
func (ext *Checker) cleanup(ctx context.Context) {
	ext.log("Evicting up any running pods with name", ext.podName())
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	checkLabelSelector := kuberhealthyCheckNameLabel + " = " + ext.CheckName
	ext.log("eviction: looking for pods with the label", checkLabelSelector)
	podList, err := podClient.List(ctx, metav1.ListOptions{
		LabelSelector: checkLabelSelector,
	})
	if err != nil {
		ext.log("error when searching for checker pods to clean up", err)
		return
	}
	wg := sync.WaitGroup{}
	for _, p := range podList.Items {
		ext.log("finding pods that are not in status.phase=Failed or status.phase=Succeeded")
		ext.log("pod:", p.Name, "is in status:", p.Status.Phase)
		if p.Status.Phase == apiv1.PodPending || p.Status.Phase == apiv1.PodUnknown || p.Status.Phase == apiv1.PodRunning {
			wg.Add(1)
			go func(p apiv1.Pod) {
				defer wg.Done()
				ext.log("evicting pod", p.GetName(), "from namespace", p.GetNamespace())
				err := ext.evictPod(ctx, p.GetName(), p.GetNamespace())
				if err != nil {
					ext.log("error killing pod", p.GetName()+":", err)
				}
			}(p)
		}
	}
	wg.Wait()
}
func (ext *Checker) evictPod(ctx context.Context, podName string, podNamespace string) error {
	podClient := ext.KubeClient.CoreV1().Pods(podNamespace)
	eviction := &policyv1.Eviction{
		ObjectMeta: metav1.ObjectMeta{
			Name:      podName,
			Namespace: podNamespace,
		},
	}
	err := podClient.Evict(ctx, eviction)
	if err != nil {
		ext.log("error when trying to cleanup/evict checker pod", podName, "in namespace", podNamespace+":", err)
		podExists, _ := util.PodNameExists(ext.KubeClient, podName, podNamespace)
		if podExists == true {
			err := util.PodKill(ext.KubeClient, podName, podNamespace, 30)
			if err != nil {
				ext.log("error killing pod", podName+":", err)
			}
		}
	}
	return err
}
func (ext *Checker) setUUID(uuid string) error {
	ext.log("Setting expected UUID to:", uuid)
	checkState, err := ext.getKHState()
	if err != nil && !(k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found")) {
		return fmt.Errorf("error setting uuid for check %s %w", ext.CheckName, err)
	}
	if err != nil && (k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found")) {
		ext.log("khstate did not exist, so a default object will be created")
		details := khstatev1.NewWorkloadDetails(ext.KHWorkload)
		details.Namespace = ext.CheckNamespace()
		details.AuthoritativePod = ext.hostname
		details.OK = true
		details.RunDuration = time.Duration(0).String()
		newState := khstatev1.NewKuberhealthyState(ext.CheckName, details)
		newState.Namespace = ext.Namespace
		ext.log("Creating khstate", newState.Name, newState.Namespace, "because it did not exist")
		_, err = ext.KHStateClient.KuberhealthyStates(ext.CheckNamespace()).Create(&newState)
		if err != nil {
			ext.log("failed to create a khstate after finding that it did not exist:", err)
			return err
		}
		checkState, err = ext.getKHState()
		if err != nil {
			ext.log("failed to fetch khstate khstate after creating it because it did not exist:", err)
			return err
		}
	}
	checkState.Spec.CurrentUUID = uuid
	ext.log("Updating khstate", checkState.Name, checkState.Namespace, "to setUUID:", checkState.Spec.CurrentUUID)
	_, err = ext.KHStateClient.KuberhealthyStates(ext.CheckNamespace()).Update(&checkState)
	tries := 0
	for {
		if tries >= 9 {
			return fmt.Errorf("failed to verify uuid match %s after %d tries", checkState.Spec.CurrentUUID, tries)
		}
		tries++
		ext.log("Waiting 1 second before checking " + ext.Name() + " uuid.")
		time.Sleep(time.Second * 1)
		extCheck, err := ext.KHStateClient.KuberhealthyStates(ext.CheckNamespace()).Get(ext.Name(), metav1.GetOptions{})
		if err != nil {
			ext.log("failed to get khstate while truing up check uuid:", err)
			continue
		}
		if checkState.Spec.CurrentUUID == extCheck.Spec.CurrentUUID {
			ext.log(ext.Name() + " verified uuid match.")
			break
		}
	}
	return err
}
func (ext *Checker) watchForCheckerPodDelete(ctx context.Context) chan error {
	ext.wg.Add(1)
	defer ext.wg.Done()
	listOptions := metav1.ListOptions{
		LabelSelector: kuberhealthyRunIDLabel + "=" + ext.currentCheckUUID,
	}
	waitForDeleteChan := make(chan error)
	watcher, err := ext.startPodWatcher(ctx, listOptions)
	if err != nil {
		waitForDeleteChan <- fmt.Errorf("error creating pod watcher: %w", err)
	}
	go func() {
		select {
		case <-ctx.Done():  
			ext.log("pod shutdown monitor stopping gracefully")
		case <-ext.waitForDeletedEvent(watcher):  
			ext.log("pod shutdown monitor witnessed the checker pod being removed")
			waitForDeleteChan <- fmt.Errorf("pod shutdown monitor witnessed the checker pod being removed")
		}
		watcher.Stop()
	}()
	return waitForDeleteChan
}
func (ext *Checker) startPodWatcher(ctx context.Context, listOptions metav1.ListOptions) (watch.Interface, error) {
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	ext.log("creating a pod watcher")
	return podClient.Watch(ctx, listOptions)
}
func (ext *Checker) waitForDeletedEvent(w watch.Interface) chan error {
	ext.wg.Add(1)
	defer ext.wg.Done()
	ext.log("starting pod shutdown watcher")
	var err error
	outChan := make(chan error, 2)
	go func() {
		for e := range w.ResultChan() {
			ext.log("got a result when watching for pod to remove")
			switch e.Type {
			case watch.Modified:  
				ext.log("checker pod shutdown monitor saw a modified event.")
				p, ok := e.Object.(*apiv1.Pod)
				if !ok {
					ext.log("checker pod shutdown monitor saw a modified event and the object was not a pod. skipped.")
					break
				}
				ext.log("checker pod shutdown monitor saw a modified event. the pod changed to ", p.Status.Phase)
			case watch.Deleted:  
				outChan <- nil
				return
			case watch.Error:
				ext.log("khcheck monitor saw an error event")
				o, ok := e.Object.(*metav1.Status)
				if !ok {
					err = fmt.Errorf("pod removal monitor had an error when watching for pod changes: " + o.String())
				} else {
					err = errors.New("unidentified error when watching for pod to be deleted")
				}
				outChan <- err
				return
			default:
				ext.log("pod removal monitor saw an irrelevant event type and ignored it:", e.Type)
			}
		}
		ext.log("wait for deleted event watcher has closed")
	}()
	ext.log("pod removal monitor ended unexpectedly")
	return outChan
}