func (exec *sidecarExecutor) monitorAWSCredsLease() {
	wait := func() {
		vars := exec.awsCredsLease
		log.Infof("Monitoring AWS Credentials expiry at '%s' for lease ID '%s'", vars.LeaseExpiryTime, vars.LeaseID)
		<-time.After(vars.LeaseExpiryTime.Sub(time.Now().UTC()))
	}
	for exec.awsCredsLease.LeaseExpiryTime.After(time.Now().UTC()) {
		wait()
	}
	pid := os.Getpid()
	ourProcess, err := os.FindProcess(pid)
	if err != nil {
		log.Errorf("FAILED attempting to shut down due to AWS lease expiration! '%s'", err)
		ourProcess, _ = os.FindProcess(pid)
	}
	log.Info("Attempting to shutdown because of AWS credential lease expiration")
	ourProcess.Signal(syscall.SIGUSR1)
}
monitorAWSCredsLease will be run in a background goroutine and will shut down the managed
process if we are about to hit our expiry. We don't bother with expiring the lease here,
it will be handled when the looper shuts down. If that somehow fails, it will still get
cleaned up by Vault afterward.
package main
import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strings"
	"sync"
	"syscall"
	"time"
	"fmt"
	"github.com/Nitro/sidecar-executor/container"
	"github.com/Nitro/sidecar-executor/vault"
	"github.com/Nitro/sidecar/service"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/jinzhu/copier"
	mesos "github.com/mesos/mesos-go/api/v1/lib"
	"github.com/relistan/go-director"
	log "github.com/sirupsen/logrus"
)
const (
	StillRunning = -1
)
type ExecDriver interface {
	NewStatus(id mesos.TaskID) mesos.TaskStatus
	SendStatusUpdate(status mesos.TaskStatus) error
	Stop()
	Run() error
}
type sidecarExecutor struct {
	client          container.DockerClient
	fetcher         SidecarFetcher
	watchLooper     director.Looper
	watcherWg       sync.WaitGroup
	logsQuitChan    chan struct{}
	dockerAuth      *docker.AuthConfiguration
	failCount       int
	vault           vault.Vault
	config          Config
	statusSleepTime time.Duration
	containerConfig *docker.CreateContainerOptions
	containerID     string
	driver          ExecDriver
	awsCredsLease   *vault.VaultAWSCredsLease
}
func newSidecarExecutor(client container.DockerClient, auth *docker.AuthConfiguration,
	config Config) *sidecarExecutor {
	var vaultConfig vault.EnvVaultConfig
	copier.Copy(&vaultConfig, &config)
	return &sidecarExecutor{
		client:          client,
		fetcher:         &http.Client{Timeout: config.HttpTimeout},
		dockerAuth:      auth,
		vault:           vault.NewDefaultVault(&vaultConfig),
		config:          config,
		statusSleepTime: DefaultStatusSleepTime,
	}
}
func (exec *sidecarExecutor) logTaskEnv(taskInfo *mesos.TaskInfo, labels map[string]string, addEnvVars []string) {
	env := container.EnvForTask(taskInfo, labels, addEnvVars)
	if len(env) < 1 {
		log.Info("No Docker environment provided")
		return
	}
	log.Infof("Docker Environment --------------------")
	for _, setting := range env {
		log.Infof(" * %s", redactSetting(setting))
	}
	log.Infof("---------------------------------------")
}
var replExpr = regexp.MustCompile("(.*)=(...).{10}(.+).{5}")
func redactSetting(setting string) string {
	if strings.Contains(setting, "SECRET") {
		return replExpr.ReplaceAllString(setting, "$1=$2[REDACTED]$3...")
	}
	return setting
}
func (exec *sidecarExecutor) sendStatus(status int64, taskID *mesos.TaskID) {
	update := exec.driver.NewStatus(*taskID)
	switch status {
	case TaskRunning:
		update.State = mesos.TASK_RUNNING.Enum()
	case TaskFinished:
		update.State = mesos.TASK_FINISHED.Enum()
	case TaskFailed:
		update.State = mesos.TASK_FAILED.Enum()
	case TaskKilled:
		update.State = mesos.TASK_KILLED.Enum()
	}
	if err := exec.driver.SendStatusUpdate(update); err != nil {
		log.Errorf("Error sending status update %s", err.Error())
		panic(err.Error())
	}
}
func (exec *sidecarExecutor) finishTask(taskInfo *mesos.TaskInfo) {
	taskID := taskInfo.GetTaskID()
	exec.sendStatus(TaskFinished, &taskID)
	time.Sleep(exec.statusSleepTime)
	exec.StopDriver()
}
func (exec *sidecarExecutor) failTask(taskInfo *mesos.TaskInfo) {
	taskID := taskInfo.GetTaskID()
	exec.sendStatus(TaskFailed, &taskID)
	time.Sleep(exec.statusSleepTime)
	exec.StopDriver()
}
func (exec *sidecarExecutor) taskKilled(taskInfo *mesos.TaskInfo) {
	taskID := taskInfo.GetTaskID()
	exec.sendStatus(TaskKilled, &taskID)
	time.Sleep(exec.statusSleepTime)
	exec.StopDriver()
}
func sidecarLookup(containerId string, services SidecarServices) (*service.Service, bool) {
	hostname := os.Getenv("TASK_HOST")  
	if _, ok := services.Servers[hostname]; !ok {
		log.Warnf("Host not found in Sidecar, can't manage this container! (%s)", hostname)
		return nil, ok
	}
	svc, ok := services.Servers[hostname].Services[containerId[:12]]
	return &svc, ok
}
func shouldBeKilled(svc *service.Service) bool {
	return svc.Status == service.UNHEALTHY || svc.Status == service.TOMBSTONE
}
func (exec *sidecarExecutor) exceededFailCount() bool {
	return exec.failCount >= exec.config.SidecarMaxFails
}
func (exec *sidecarExecutor) sidecarStatus(containerId string) error {
	fetch := func() ([]byte, error) {
		resp, err := exec.fetcher.Get(exec.config.SidecarUrl)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		return body, nil
	}
	var err error
	var data []byte
	for i := 0; i <= exec.config.SidecarRetryCount; i++ {
		data, err = fetch()
		if err == nil {
			break
		}
		log.Warnf("Failed %d attempts to fetch state from Sidecar!", i+1)
		time.Sleep(exec.config.SidecarRetryDelay)
	}
	if err != nil {
		log.Error("Can't contact Sidecar! Assuming healthy...")
		return nil
	}
	var services SidecarServices
	err = json.Unmarshal(data, &services)
	if err != nil {
		log.Error("Can't parse Sidecar results! Assuming healthy...")
		return nil
	}
	svc, ok := sidecarLookup(containerId, services)
	if !ok {
		log.Errorf("Can't find this service in Sidecar yet! Assuming healthy...")
		return nil
	}
	if shouldBeKilled(svc) {
		if !exec.exceededFailCount() {
			exec.failCount += 1
			log.Warnf("Failed Sidecar health check, but below fail limit")
			return nil
		}
		log.Errorf("Health failure count exceeded %d", exec.config.SidecarMaxFails)
		exec.failCount = 0
		return errors.New("Unhealthy container: " + containerId + " failing task!")
	}
	exec.failCount = 0  
	return nil
}
func (exec *sidecarExecutor) monitorTask(cntnrId string, taskInfo *mesos.TaskInfo, checkSidecar bool) {
	log.Infof("Monitoring Mesos task %s for container %s [checkSidecar: %t]",
		taskInfo.TaskID.Value, cntnrId[:12], checkSidecar,
	)
	if checkSidecar {
		time.Sleep(exec.config.SidecarBackoff)
	}
	exec.watcherWg.Add(1)
	containerName := container.GetContainerName(&taskInfo.TaskID)
	var exitCode int = StillRunning
	go exec.watchLooper.Loop(func() error {
		var err error
		exitCode, err = exec.checkContainerStatus(cntnrId, checkSidecar)
		return err
	})
	err := exec.watchLooper.Wait()
	if err != nil {
		log.Errorf("Error! %s", err)
	}
	if exitCode == StillRunning {
		err := container.StopContainer(
			exec.client, containerName, exec.config.KillTaskTimeout,
		)
		if err != nil {
			log.Errorf("Error stopping container %s! %s", containerName, err)
		}
	}
	if exitCode == StillRunning {
		exitCode, err = exec.checkContainerStatus(cntnrId, checkSidecar)
		if err != nil {
			log.Error("Unable to check container status! Assuming dead, moving on.")
		}
	}
	exec.maybeCleanupAWSCredsLease()
	err = exec.vault.MaybeRevokeToken()
	if err != nil {
		log.Error(err.Error())
	}
	exec.handleContainerExit(taskInfo, exitCode)
	exec.watcherWg.Done()
}
func (exec *sidecarExecutor) handleContainerExit(taskInfo *mesos.TaskInfo, exitCode int) {
	if exitCode != 0 {
		containerName := container.GetContainerName(&taskInfo.TaskID)
		exec.copyLogs(containerName)
	}
	switch {
	case exitCode > 128 && exitCode <= 165:
		log.Error("Task was killed, notifying Mesos")
		exec.taskKilled(taskInfo)
	case exitCode > 0:  
		log.Error("Task failed, notifying Mesos")
		exec.failTask(taskInfo)
	case exitCode < 0:  
		log.Error("Task may still be running despite attempts to kill!")
		exec.failTask(taskInfo)
	default:
		log.Info("Task completed: ", taskInfo.GetName())
		exec.finishTask(taskInfo)
	}
}
func (exec *sidecarExecutor) maybeCleanupAWSCredsLease() {
	if exec.awsCredsLease == nil {
		log.Info("No AWS lease to clean up, skipping")
		return
	}
	err := exec.vault.RevokeAWSCredsLease(exec.awsCredsLease.LeaseID, exec.awsCredsLease.Role)
	if err != nil {
		log.Errorf("Unable to revoke AWS Creds Lease: %s", err)
	}
}
func (exec *sidecarExecutor) checkContainerStatus(containerId string, checkSidecar bool) (int, error) {
	containers, err := exec.client.ListContainers(
		docker.ListContainersOptions{},
	)
	if err != nil {
		return StillRunning, err
	}
	if !containerIsPresent(containers, containerId) {
		exec.watchLooper.Quit()  
		exitCode, err := container.GetExitCode(exec.client, containerId)
		if err != nil {
			return StillRunning, err
		}
		msg := fmt.Sprintf("Container %s not running! - ExitCode: %d", containerId, exitCode)
		if exitCode == 0 {
			log.Info(msg)
			return 0, nil
		}
		return exitCode, errors.New(msg)
	}
	return StillRunning, exec.maybeCheckSidecar(containerId, checkSidecar)
}
func (exec *sidecarExecutor) maybeCheckSidecar(containerId string, checkSidecar bool) error {
	if !checkSidecar {
		return nil
	}
	return exec.sidecarStatus(containerId)
}
func containerIsPresent(containers []docker.APIContainers, containerId string) bool {
	for _, entry := range containers {
		if entry.ID == containerId {
			return true
		}
	}
	return false
}
func (exec *sidecarExecutor) maybePullContainer(taskInfo *mesos.TaskInfo) error {
	var shouldPullContainer bool
	if !container.CheckImage(exec.client, taskInfo) {
		shouldPullContainer = true
	}
	if taskInfo.Container.Docker.ForcePullImage != nil && *taskInfo.Container.Docker.ForcePullImage {
		shouldPullContainer = true
	}
	log.Infof("Using image '%s'", taskInfo.Container.Docker.Image)
	if shouldPullContainer {
		err := container.PullImage(exec.client, taskInfo, exec.dockerAuth)
		if err != nil {
			return err
		}
	}
	log.Info("Re-using existing image... already present")
	return nil
}