func (st *HelmState) SyncReleases(affectedReleases *AffectedReleases, helm helmexec.Interface, additionalValues []string, workerLimit int, opt ...SyncOpt) []error {
	opts := &SyncOpts{}
	for _, o := range opt {
		o.Apply(opts)
	}
	preps, prepErrs := st.prepareSyncReleases(helm, additionalValues, workerLimit, opts)
	defer func() {
		if opts.SkipCleanup {
			return
		}
		for _, p := range preps {
			st.removeFiles(p.files)
		}
	}()
	if len(prepErrs) > 0 {
		return prepErrs
	}
	errs := []error{}
	jobQueue := make(chan *syncPrepareResult, len(preps))
	results := make(chan syncResult, len(preps))
	if workerLimit == 0 {
		workerLimit = len(preps)
	}
	m := new(sync.Mutex)
	st.scatterGather(
		workerLimit,
		len(preps),
		func() {
			for i := 0; i < len(preps); i++ {
				jobQueue <- &preps[i]
			}
			close(jobQueue)
		},
		func(workerIndex int) {
			for prep := range jobQueue {
				release := prep.release
				flags := prep.flags
				chart := normalizeChart(st.basePath, release.Chart)
				var relErr *ReleaseError
				context := st.createHelmContext(release, workerIndex)
				if _, err := st.triggerPresyncEvent(release, "sync"); err != nil {
					relErr = newReleaseFailedError(release, err)
				} else if !release.Desired() {
					installed, err := st.isReleaseInstalled(context, helm, *release)
					if err != nil {
						relErr = newReleaseFailedError(release, err)
					} else if installed {
						var args []string
						if helm.IsHelm3() {
							args = []string{}
						} else {
							args = []string{"--purge"}
						}
						deletionFlags := st.appendConnectionFlags(args, helm, release)
						m.Lock()
						if _, err := st.triggerReleaseEvent("preuninstall", nil, release, "sync"); err != nil {
							affectedReleases.Failed = append(affectedReleases.Failed, release)
							relErr = newReleaseFailedError(release, err)
						} else if err := helm.DeleteRelease(context, release.Name, deletionFlags...); err != nil {
							affectedReleases.Failed = append(affectedReleases.Failed, release)
							relErr = newReleaseFailedError(release, err)
						} else if _, err := st.triggerReleaseEvent("postuninstall", nil, release, "sync"); err != nil {
							affectedReleases.Failed = append(affectedReleases.Failed, release)
							relErr = newReleaseFailedError(release, err)
						} else {
							affectedReleases.Deleted = append(affectedReleases.Deleted, release)
						}
						m.Unlock()
					}
				} else if err := helm.SyncRelease(context, release.Name, chart, flags...); err != nil {
					m.Lock()
					affectedReleases.Failed = append(affectedReleases.Failed, release)
					m.Unlock()
					relErr = newReleaseFailedError(release, err)
				} else {
					m.Lock()
					affectedReleases.Upgraded = append(affectedReleases.Upgraded, release)
					m.Unlock()
					installedVersion, err := st.getDeployedVersion(context, helm, release)
					if err != nil {  
						st.logger.Debugf("getting deployed release version failed:%v", err)
					} else {
						release.installedVersion = installedVersion
					}
				}
				if _, err := st.triggerPostsyncEvent(release, relErr, "sync"); err != nil {
					st.logger.Warnf("warn: %v\n", err)
				}
				if _, err := st.TriggerCleanupEvent(release, "sync"); err != nil {
					st.logger.Warnf("warn: %v\n", err)
				}
				if relErr == nil {
					results <- syncResult{}
				} else {
					results <- syncResult{errors: []*ReleaseError{relErr}}
				}
			}
		},
		func() {
			for i := 0; i < len(preps); {
				select {
				case res := <-results:
					if len(res.errors) > 0 {
						for _, e := range res.errors {
							errs = append(errs, e)
						}
					}
				}
				i++
			}
		},
	)
	if len(errs) > 0 {
		return errs
	}
	return nil
}
SyncReleases wrapper for executing helm upgrade on the releases
package state
import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"text/template"
	"github.com/imdario/mergo"
	"github.com/variantdev/chartify"
	"github.com/roboll/helmfile/pkg/environment"
	"github.com/roboll/helmfile/pkg/event"
	"github.com/roboll/helmfile/pkg/helmexec"
	"github.com/roboll/helmfile/pkg/remote"
	"github.com/roboll/helmfile/pkg/tmpl"
	"github.com/tatsushid/go-prettytable"
	"github.com/variantdev/vals"
	"go.uber.org/zap"
	"gopkg.in/yaml.v2"
)
const (
	EmptyTimeout = -1
)
type ReleaseSetSpec struct {
	DefaultHelmBinary string `yaml:"helmBinary,omitempty"`
	DefaultValues []interface{} `yaml:"values,omitempty"`
	Environments map[string]EnvironmentSpec `yaml:"environments,omitempty"`
	Bases               []string          `yaml:"bases,omitempty"`
	HelmDefaults        HelmSpec          `yaml:"helmDefaults,omitempty"`
	Helmfiles           []SubHelmfileSpec `yaml:"helmfiles,omitempty"`
	DeprecatedContext   string            `yaml:"context,omitempty"`
	DeprecatedReleases  []ReleaseSpec     `yaml:"charts,omitempty"`
	OverrideKubeContext string            `yaml:"kubeContext,omitempty"`
	OverrideNamespace   string            `yaml:"namespace,omitempty"`
	OverrideChart       string            `yaml:"chart,omitempty"`
	Repositories        []RepositorySpec  `yaml:"repositories,omitempty"`
	CommonLabels        map[string]string `yaml:"commonLabels,omitempty"`
	Releases            []ReleaseSpec     `yaml:"releases,omitempty"`
	Selectors           []string          `yaml:"-"`
	ApiVersions         []string          `yaml:"apiVersions,omitempty"`
	Hooks []event.Hook `yaml:"hooks,omitempty"`
	Templates map[string]TemplateSpec `yaml:"templates"`
	Env environment.Environment `yaml:"-"`
	MissingFileHandler string `yaml:"missingFileHandler,omitempty"`
}
type PullCommand struct {
	ChartRef     string
	responseChan chan error
}
type HelmState struct {
	basePath string
	FilePath string
	ReleaseSetSpec `yaml:",inline"`
	logger *zap.SugaredLogger
	readFile          func(string) ([]byte, error)
	removeFile        func(string) error
	fileExists        func(string) (bool, error)
	glob              func(string) ([]string, error)
	tempDir           func(string, string) (string, error)
	directoryExistsAt func(string) bool
	runner      helmexec.Runner
	valsRuntime vals.Evaluator
	RenderedValues map[string]interface{}
}
type SubHelmfileSpec struct {
	Path string `yaml:"path,omitempty"`
	Selectors []string `yaml:"selectors,omitempty"`
	SelectorsInherited bool `yaml:"selectorsInherited,omitempty"`
	Environment SubhelmfileEnvironmentSpec
}
type SubhelmfileEnvironmentSpec struct {
	OverrideValues []interface{} `yaml:"values,omitempty"`
}
type HelmSpec struct {
	KubeContext     string   `yaml:"kubeContext,omitempty"`
	TillerNamespace string   `yaml:"tillerNamespace,omitempty"`
	Tillerless      bool     `yaml:"tillerless"`
	Args            []string `yaml:"args,omitempty"`
	Verify          bool     `yaml:"verify"`
	Devel bool `yaml:"devel"`
	Wait bool `yaml:"wait"`
	WaitForJobs bool `yaml:"waitForJobs"`
	Timeout int `yaml:"timeout"`
	RecreatePods bool `yaml:"recreatePods"`
	Force bool `yaml:"force"`
	Atomic bool `yaml:"atomic"`
	CleanupOnFail bool `yaml:"cleanupOnFail,omitempty"`
	HistoryMax *int `yaml:"historyMax,omitempty"`
	CreateNamespace *bool `yaml:"createNamespace,omitempty"`
	SkipDeps bool `yaml:"skipDeps"`
	TLS                      bool   `yaml:"tls"`
	TLSCACert                string `yaml:"tlsCACert,omitempty"`
	TLSKey                   string `yaml:"tlsKey,omitempty"`
	TLSCert                  string `yaml:"tlsCert,omitempty"`
	DisableValidation        *bool  `yaml:"disableValidation,omitempty"`
	DisableOpenAPIValidation *bool  `yaml:"disableOpenAPIValidation,omitempty"`
}
type RepositorySpec struct {
	Name            string `yaml:"name,omitempty"`
	URL             string `yaml:"url,omitempty"`
	CaFile          string `yaml:"caFile,omitempty"`
	CertFile        string `yaml:"certFile,omitempty"`
	KeyFile         string `yaml:"keyFile,omitempty"`
	Username        string `yaml:"username,omitempty"`
	Password        string `yaml:"password,omitempty"`
	Managed         string `yaml:"managed,omitempty"`
	OCI             bool   `yaml:"oci,omitempty"`
	PassCredentials string `yaml:"passCredentials,omitempty"`
}
type ReleaseSpec struct {
	Chart string `yaml:"chart,omitempty"`
	Directory string `yaml:"directory,omitempty"`
	Version string `yaml:"version,omitempty"`
	Verify *bool `yaml:"verify,omitempty"`
	Devel *bool `yaml:"devel,omitempty"`
	Wait *bool `yaml:"wait,omitempty"`
	WaitForJobs *bool `yaml:"waitForJobs,omitempty"`
	Timeout *int `yaml:"timeout,omitempty"`
	RecreatePods *bool `yaml:"recreatePods,omitempty"`
	Force *bool `yaml:"force,omitempty"`
	Installed *bool `yaml:"installed,omitempty"`
	Atomic *bool `yaml:"atomic,omitempty"`
	CleanupOnFail *bool `yaml:"cleanupOnFail,omitempty"`
	HistoryMax *int `yaml:"historyMax,omitempty"`
	Condition string `yaml:"condition,omitempty"`
	CreateNamespace *bool `yaml:"createNamespace,omitempty"`
	DisableOpenAPIValidation *bool `yaml:"disableOpenAPIValidation,omitempty"`
	DisableValidation *bool `yaml:"disableValidation,omitempty"`
	DisableValidationOnInstall *bool `yaml:"disableValidationOnInstall,omitempty"`
	MissingFileHandler *string `yaml:"missingFileHandler,omitempty"`
	Needs []string `yaml:"needs,omitempty"`
	Hooks []event.Hook `yaml:"hooks,omitempty"`
	Name      string            `yaml:"name,omitempty"`
	Namespace string            `yaml:"namespace,omitempty"`
	Labels    map[string]string `yaml:"labels,omitempty"`
	Values    []interface{}     `yaml:"values,omitempty"`
	Secrets   []interface{}     `yaml:"secrets,omitempty"`
	SetValues []SetValue        `yaml:"set,omitempty"`
	ValuesTemplate    []interface{} `yaml:"valuesTemplate,omitempty"`
	SetValuesTemplate []SetValue    `yaml:"setTemplate,omitempty"`
	EnvValues []SetValue `yaml:"env,omitempty"`
	ValuesPathPrefix string `yaml:"valuesPathPrefix,omitempty"`
	TillerNamespace string `yaml:"tillerNamespace,omitempty"`
	Tillerless      *bool  `yaml:"tillerless,omitempty"`
	KubeContext string `yaml:"kubeContext,omitempty"`
	TLS       *bool  `yaml:"tls,omitempty"`
	TLSCACert string `yaml:"tlsCACert,omitempty"`
	TLSKey    string `yaml:"tlsKey,omitempty"`
	TLSCert   string `yaml:"tlsCert,omitempty"`
	TillerlessTemplate *string `yaml:"tillerlessTemplate,omitempty"`
	VerifyTemplate     *string `yaml:"verifyTemplate,omitempty"`
	WaitTemplate       *string `yaml:"waitTemplate,omitempty"`
	InstalledTemplate  *string `yaml:"installedTemplate,omitempty"`
	Dependencies          []Dependency  `yaml:"dependencies,omitempty"`
	JSONPatches           []interface{} `yaml:"jsonPatches,omitempty"`
	StrategicMergePatches []interface{} `yaml:"strategicMergePatches,omitempty"`
	Transformers []interface{} `yaml:"transformers,omitempty"`
	Adopt        []string      `yaml:"adopt,omitempty"`
	installedVersion string
	ForceGoGetter bool `yaml:"forceGoGetter,omitempty"`
	ForceNamespace string `yaml:"forceNamespace,omitempty"`
	SkipDeps *bool `yaml:"skipDeps,omitempty"`
}
type Release struct {
	ReleaseSpec
	Filtered bool
}
type SetValue struct {
	Name   string   `yaml:"name,omitempty"`
	Value  string   `yaml:"value,omitempty"`
	File   string   `yaml:"file,omitempty"`
	Values []string `yaml:"values,omitempty"`
}
type AffectedReleases struct {
	Upgraded []*ReleaseSpec
	Deleted  []*ReleaseSpec
	Failed   []*ReleaseSpec
}
const DefaultEnv = "default"
const MissingFileHandlerError = "Error"
const MissingFileHandlerInfo = "Info"
const MissingFileHandlerWarn = "Warn"
const MissingFileHandlerDebug = "Debug"
func (st *HelmState) ApplyOverrides(spec *ReleaseSpec) {
	if st.OverrideKubeContext != "" {
		spec.KubeContext = st.OverrideKubeContext
	}
	if st.OverrideNamespace != "" {
		spec.Namespace = st.OverrideNamespace
	}
	var needs []string
	for i := 0; i < len(spec.Needs); i++ {
		n := spec.Needs[i]
		var kubecontext, ns, name string
		components := strings.Split(n, "/")
		name = components[len(components)-1]
		if len(components) > 1 {
			ns = components[len(components)-2]
		} else if spec.TillerNamespace != "" {
			ns = spec.TillerNamespace
		} else {
			ns = spec.Namespace
		}
		if len(components) > 2 {
			kubecontext = components[len(components)-3]
		} else {
			kubecontext = spec.KubeContext
		}
		var componentsAfterOverride []string
		if kubecontext != "" {
			componentsAfterOverride = append(componentsAfterOverride, kubecontext)
		}
		if kubecontext != "" || ns != "" {
			componentsAfterOverride = append(componentsAfterOverride, ns)
		}
		componentsAfterOverride = append(componentsAfterOverride, name)
		needs = append(needs, strings.Join(componentsAfterOverride, "/"))
	}
	spec.Needs = needs
}
type RepoUpdater interface {
	IsHelm3() bool
	AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string, passCredentials string) error
	UpdateRepo() error
	RegistryLogin(name string, username string, password string) error
}
func (st *HelmState) SyncRepos(helm RepoUpdater, shouldSkip map[string]bool) ([]string, error) {
	var updated []string
	for _, repo := range st.Repositories {
		if shouldSkip[repo.Name] {
			continue
		}
		var err error
		if repo.OCI {
			username, password := gatherOCIUsernamePassword(repo.Name, repo.Username, repo.Password)
			if username != "" && password != "" {
				err = helm.RegistryLogin(repo.URL, username, password)
			}
		} else {
			err = helm.AddRepo(repo.Name, repo.URL, repo.CaFile, repo.CertFile, repo.KeyFile, repo.Username, repo.Password, repo.Managed, repo.PassCredentials)
		}
		if err != nil {
			return nil, err
		}
		updated = append(updated, repo.Name)
	}
	return updated, nil
}
func gatherOCIUsernamePassword(repoName string, username string, password string) (string, string) {
	var user, pass string
	if username != "" {
		user = username
	} else if u := os.Getenv(fmt.Sprintf("%s_USERNAME", strings.ToUpper(repoName))); u != "" {
		user = u
	}
	if password != "" {
		pass = password
	} else if p := os.Getenv(fmt.Sprintf("%s_PASSWORD", strings.ToUpper(repoName))); p != "" {
		pass = p
	}
	return user, pass
}
type syncResult struct {
	errors []*ReleaseError
}
type syncPrepareResult struct {
	release *ReleaseSpec
	flags   []string
	errors  []*ReleaseError
	files   []string
}
func (st *HelmState) prepareSyncReleases(helm helmexec.Interface, additionalValues []string, concurrency int, opt ...SyncOpt) ([]syncPrepareResult, []error) {
	opts := &SyncOpts{}
	for _, o := range opt {
		o.Apply(opts)
	}
	releases := []*ReleaseSpec{}
	for i, _ := range st.Releases {
		releases = append(releases, &st.Releases[i])
	}
	numReleases := len(releases)
	jobs := make(chan *ReleaseSpec, numReleases)
	results := make(chan syncPrepareResult, numReleases)
	res := []syncPrepareResult{}
	errs := []error{}
	mut := sync.Mutex{}
	st.scatterGather(
		concurrency,
		numReleases,
		func() {
			for i := 0; i < numReleases; i++ {
				jobs <- releases[i]
			}
			close(jobs)
		},
		func(workerIndex int) {
			for release := range jobs {
				st.ApplyOverrides(release)
				if !release.Desired() {
					results <- syncPrepareResult{release: release, flags: []string{}, errors: []*ReleaseError{}}
					continue
				}
				mut.Lock()
				flags, files, flagsErr := st.flagsForUpgrade(helm, release, workerIndex)
				mut.Unlock()
				if flagsErr != nil {
					results <- syncPrepareResult{errors: []*ReleaseError{newReleaseFailedError(release, flagsErr)}, files: files}
					continue
				}
				errs := []*ReleaseError{}
				for _, value := range additionalValues {
					valfile, err := filepath.Abs(value)
					if err != nil {
						errs = append(errs, newReleaseFailedError(release, err))
					}
					ok, err := st.fileExists(valfile)
					if err != nil {
						errs = append(errs, newReleaseFailedError(release, err))
					} else if !ok {
						errs = append(errs, newReleaseFailedError(release, fmt.Errorf("file does not exist: %s", valfile)))
					}
					flags = append(flags, "--values", valfile)
				}
				if opts.Set != nil {
					for _, s := range opts.Set {
						flags = append(flags, "--set", s)
					}
				}
				if opts.SkipCRDs {
					flags = append(flags, "--skip-crds")
				}
				if opts.Wait {
					flags = append(flags, "--wait")
				}
				if opts.WaitForJobs {
					flags = append(flags, "--wait-for-jobs")
				}
				if len(errs) > 0 {
					results <- syncPrepareResult{errors: errs, files: files}
					continue
				}
				results <- syncPrepareResult{release: release, flags: flags, errors: []*ReleaseError{}, files: files}
			}
		},
		func() {
			for i := 0; i < numReleases; {
				select {
				case r := <-results:
					for _, e := range r.errors {
						errs = append(errs, e)
					}
					res = append(res, r)
					i++
				}
			}
		},
	)
	return res, errs
}
func (st *HelmState) isReleaseInstalled(context helmexec.HelmContext, helm helmexec.Interface, release ReleaseSpec) (bool, error) {
	out, err := st.listReleases(context, helm, &release)
	if err != nil {
		return false, err
	} else if out != "" {
		return true, nil
	}
	return false, nil
}
func (st *HelmState) DetectReleasesToBeDeletedForSync(helm helmexec.Interface, releases []ReleaseSpec) ([]ReleaseSpec, error) {
	detected := []ReleaseSpec{}
	for i := range releases {
		release := releases[i]
		if !release.Desired() {
			installed, err := st.isReleaseInstalled(st.createHelmContext(&release, 0), helm, release)
			if err != nil {
				return nil, err
			} else if installed {
				r := release
				detected = append(detected, r)
			}
		}
	}
	return detected, nil
}
func (st *HelmState) DetectReleasesToBeDeleted(helm helmexec.Interface, releases []ReleaseSpec) ([]ReleaseSpec, error) {
	detected := []ReleaseSpec{}
	for i := range releases {
		release := releases[i]
		installed, err := st.isReleaseInstalled(st.createHelmContext(&release, 0), helm, release)
		if err != nil {
			return nil, err
		} else if installed {
			r := release
			detected = append(detected, r)
		}
	}
	return detected, nil
}
type SyncOpts struct {
	Set         []string
	SkipCleanup bool
	SkipCRDs    bool
	Wait        bool
	WaitForJobs bool
}
type SyncOpt interface{ Apply(*SyncOpts) }
func (o *SyncOpts) Apply(opts *SyncOpts) {
	*opts = *o
}
func ReleaseToID(r *ReleaseSpec) string {
	var id string
	kc := r.KubeContext
	if kc != "" {
		id += kc + "/"
	}
	tns := r.TillerNamespace
	ns := r.Namespace
	if tns != "" {
		id += tns + "/"
	} else if ns != "" {
		id += ns + "/"
	}
	if kc != "" {
		if tns == "" && ns == "" {
			id += "/"
		}
	}
	id += r.Name
	return id
}
func (st *HelmState) DeleteReleasesForSync(affectedReleases *AffectedReleases, helm helmexec.Interface, workerLimit int) []error {
	errs := []error{}
	releases := st.Releases
	jobQueue := make(chan *ReleaseSpec, len(releases))
	results := make(chan syncResult, len(releases))
	if workerLimit == 0 {
		workerLimit = len(releases)
	}
	m := new(sync.Mutex)
	st.scatterGather(
		workerLimit,
		len(releases),
		func() {
			for i := 0; i < len(releases); i++ {
				jobQueue <- &releases[i]
			}
			close(jobQueue)
		},
		func(workerIndex int) {
			for release := range jobQueue {
				var relErr *ReleaseError
				context := st.createHelmContext(release, workerIndex)
				if _, err := st.triggerPresyncEvent(release, "sync"); err != nil {
					relErr = newReleaseFailedError(release, err)
				} else {
					var args []string
					if helm.IsHelm3() {
						args = []string{}
						if release.Namespace != "" {
							args = append(args, "--namespace", release.Namespace)
						}
					} else {
						args = []string{"--purge"}
					}
					deletionFlags := st.appendConnectionFlags(args, helm, release)
					m.Lock()
					if _, err := st.triggerReleaseEvent("preuninstall", nil, release, "sync"); err != nil {
						affectedReleases.Failed = append(affectedReleases.Failed, release)
						relErr = newReleaseFailedError(release, err)
					} else if err := helm.DeleteRelease(context, release.Name, deletionFlags...); err != nil {
						affectedReleases.Failed = append(affectedReleases.Failed, release)
						relErr = newReleaseFailedError(release, err)
					} else if _, err := st.triggerReleaseEvent("postuninstall", nil, release, "sync"); err != nil {
						affectedReleases.Failed = append(affectedReleases.Failed, release)
						relErr = newReleaseFailedError(release, err)
					} else {
						affectedReleases.Deleted = append(affectedReleases.Deleted, release)
					}
					m.Unlock()
				}
				if _, err := st.triggerPostsyncEvent(release, relErr, "sync"); err != nil {
					st.logger.Warnf("warn: %v\n", err)
				}
				if _, err := st.TriggerCleanupEvent(release, "sync"); err != nil {
					st.logger.Warnf("warn: %v\n", err)
				}
				if relErr == nil {
					results <- syncResult{}
				} else {
					results <- syncResult{errors: []*ReleaseError{relErr}}
				}
			}
		},
		func() {
			for i := 0; i < len(releases); {
				select {
				case res := <-results:
					if len(res.errors) > 0 {
						for _, e := range res.errors {
							errs = append(errs, e)
						}
					}
				}
				i++
			}
		},
	)
	if len(errs) > 0 {
		return errs
	}
	return nil
}