func (a *App) loadComponents() (map[string]Component, error) {
	var list []Component
	loadDirComponents := func(dir string) error {
		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if path == dir {
				return nil
			}
			if info.IsDir() {
				files, err := filepath.Glob(filepath.Join(path, "*"))
				if err != nil {
					return err
				}
				var staticFiles []string
				hasIndexJsonnet := false
				hasIndexYAML := false
				for _, f := range files {
					stat, err := os.Stat(f)
					if err != nil {
						return err
					}
					if stat.IsDir() {
						continue
					}
					switch filepath.Base(f) {
					case "index.jsonnet":
						hasIndexJsonnet = true
					case "index.yaml":
						hasIndexYAML = true
					}
					if strings.HasSuffix(f, ".json") || strings.HasSuffix(f, ".yaml") {
						staticFiles = append(staticFiles, f)
					}
				}
				switch {
				case hasIndexJsonnet:
					list = append(list, Component{
						Name:  filepath.Base(path),
						Files: []string{filepath.Join(path, "index.jsonnet")},
					})
				case hasIndexYAML:
					list = append(list, Component{
						Name:  filepath.Base(path),
						Files: staticFiles,
					})
				}
				return filepath.SkipDir
			}
			extension := filepath.Ext(path)
			if supportedExtensions[extension] {
				list = append(list, Component{
					Name:  strings.TrimSuffix(filepath.Base(path), extension),
					Files: []string{path},
				})
			}
			return nil
		})
		return err
	}
	ds, err := filepath.Glob(a.inner.Spec.ComponentsDir)
	if err != nil {
		return nil, err
	}
	var dirs []string
	for _, d := range ds {
		s, err := os.Stat(d)
		if err != nil {
			return nil, err
		}
		if s.IsDir() {
			dirs = append(dirs, d)
		}
	}
	if len(dirs) == 0 {
		return nil, fmt.Errorf("no component directories found after expanding %s", a.inner.Spec.ComponentsDir)
	}
	for _, d := range dirs {
		err := loadDirComponents(d)
		if err != nil {
			return nil, err
		}
	}
	m := make(map[string]Component, len(list))
	for _, c := range list {
		if old, ok := m[c.Name]; ok {
			return nil, fmt.Errorf("duplicate component %s, found %s and %s", c.Name, old.Files[0], c.Files[0])
		}
		m[c.Name] = c
	}
	return m, nil
}
loadComponents loads metadata for all components for the app. It first expands the components directory
for glob patterns and loads components from all directories that match. It does _not_ recurse
into subdirectories. The data is returned as a map keyed by component name.
Note that component names must be unique across all directories. Support for multiple directories is just a
way to partition classes of components and does not introduce any namespace semantics.
package model
import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"
	"github.com/ghodss/yaml"
	"github.com/pkg/errors"
	"github.com/splunk/qbec/internal/filematcher"
	"github.com/splunk/qbec/internal/sio"
)
const Baseline = "_"
const (
	DefaultComponentsDir = "components"        
	DefaultParamsFile    = "params.libsonnet"  
)
var supportedExtensions = map[string]bool{
	".jsonnet": true,
	".yaml":    true,
	".json":    true,
}
var httpClient = &http.Client{Timeout: time.Second * 10}
type Component struct {
	Name         string    
	Files        []string  
	TopLevelVars []string  
}
type App struct {
	inner             QbecApp               
	overrideNs        string                
	tag               string                
	root              string                
	allComponents     map[string]Component  
	defaultComponents map[string]Component  
}
func makeValError(file string, errs []error) error {
	var msgs []string
	for _, err := range errs {
		msgs = append(msgs, err.Error())
	}
	return fmt.Errorf("file: %s, %d schema validation error(s): %s", file, len(errs), strings.Join(msgs, "\n"))
}
func downloadEnvFile(url string) ([]byte, error) {
	res, err := httpClient.Get(url)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("status : %s", res.Status)
	}
	payload, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	return payload, err
}
func readEnvFile(file string) ([]byte, error) {
	if filematcher.IsRemoteFile(file) {
		b, err := downloadEnvFile(file)
		if err != nil {
			return nil, errors.Wrapf(err, "download environments from %s", file)
		}
		return b, nil
	}
	return ioutil.ReadFile(file)
}
func loadEnvFiles(app *QbecApp, additionalFiles []string, v *validator) error {
	if app.Spec.Environments == nil {
		app.Spec.Environments = map[string]Environment{}
	}
	sources := map[string]string{}
	for k := range app.Spec.Environments {
		sources[k] = "inline"
	}
	var envFiles []string
	envFiles = append(envFiles, app.Spec.EnvFiles...)
	envFiles = append(envFiles, additionalFiles...)
	var allFiles []string
	for _, filePattern := range envFiles {
		matchedFiles, err := filematcher.Match(filePattern)
		if err != nil {
			return err
		}
		allFiles = append(allFiles, matchedFiles...)
	}
	for _, file := range allFiles {
		b, err := readEnvFile(file)
		if err != nil {
			return err
		}
		var qEnvs QbecEnvironmentMap
		if err := yaml.Unmarshal(b, &qEnvs); err != nil {
			return errors.Wrap(err, fmt.Sprintf("%s: unmarshal YAML", file))
		}
		errs := v.validateEnvYAML(b)
		if len(errs) > 0 {
			return makeValError(file, errs)
		}
		for k, v := range qEnvs.Spec.Environments {
			old, ok := sources[k]
			if ok {
				sio.Warnf("override env definition '%s' from file %s (previous: %s)\n", k, file, old)
			}
			sources[k] = file
			app.Spec.Environments[k] = v
		}
	}
	return nil
}
func NewApp(file string, envFiles []string, tag string) (*App, error) {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	var qApp QbecApp
	if err := yaml.Unmarshal(b, &qApp); err != nil {
		return nil, errors.Wrap(err, "unmarshal YAML")
	}
	v, err := newValidator()
	if err != nil {
		return nil, errors.Wrap(err, "create schema validator")
	}
	errs := v.validateYAML(b)
	if len(errs) > 0 {
		return nil, makeValError(file, errs)
	}
	if err := loadEnvFiles(&qApp, envFiles, v); err != nil {
		return nil, err
	}
	if len(qApp.Spec.Environments) == 0 {
		return nil, fmt.Errorf("%s: no environments defined for app", file)
	}
	for name, env := range qApp.Spec.Environments {
		if err := env.assertValid(); err != nil {
			return nil, errors.Wrapf(err, "verify environment %s", name)
		}
	}
	app := App{inner: qApp}
	dir := filepath.Dir(file)
	if !filepath.IsAbs(dir) {
		var err error
		dir, err = filepath.Abs(dir)
		if err != nil {
			return nil, errors.Wrap(err, "abs path for "+dir)
		}
	}
	app.root = dir
	app.setupDefaults()
	app.allComponents, err = app.loadComponents()
	if err != nil {
		return nil, errors.Wrap(err, "load components")
	}
	if err := app.verifyEnvAndComponentReferences(); err != nil {
		return nil, err
	}
	if err := app.verifyVariables(); err != nil {
		return nil, err
	}
	if err := app.verifyProcessors(); err != nil {
		return nil, err
	}
	app.updateComponentTopLevelVars()
	app.defaultComponents = make(map[string]Component, len(app.allComponents))
	for k, v := range app.allComponents {
		app.defaultComponents[k] = v
	}
	for _, k := range app.inner.Spec.Excludes {
		delete(app.defaultComponents, k)
	}
	if tag != "" {
		if !reLabelValue.MatchString(tag) {
			return nil, fmt.Errorf("invalid tag name '%s', must match %v", tag, reLabelValue)
		}
	}
	app.tag = tag
	return &app, nil
}
func (a *App) SetOverrideNamespace(ns string) {
	if ns != "" {
		sio.Warnln("force default namespace to", ns)
	}
	a.overrideNs = ns
}
func (a *App) setupDefaults() {
	if a.inner.Spec.ComponentsDir == "" {
		a.inner.Spec.ComponentsDir = DefaultComponentsDir
	}
	if a.inner.Spec.ParamsFile == "" {
		a.inner.Spec.ParamsFile = DefaultParamsFile
	}
}
func (a *App) Name() string {
	return a.inner.Metadata.Name
}
func (a *App) Tag() string {
	return a.tag
}
func (a *App) ParamsFile() string {
	return a.inner.Spec.ParamsFile
}
func splitPath(s string) []string {
	if s == "" {
		return nil
	}
	return strings.Split(s, ":")
}
func (a *App) PostProcessors() []string {
	return splitPath(a.inner.Spec.PostProcessor)
}
func (a *App) LibPaths() []string {
	return a.inner.Spec.LibPaths
}
func (a *App) AddComponentLabel() bool {
	return a.inner.Spec.AddComponentLabel
}
func (a *App) envObject(env string) (Environment, error) {
	envObj, ok := a.inner.Spec.Environments[env]
	if !ok {
		return envObj, fmt.Errorf("invalid environment %q", env)
	}
	return envObj, nil
}
func (a *App) ServerURL(env string) (string, error) {
	e, err := a.envObject(env)
	if err != nil {
		return "", err
	}
	return e.Server, nil
}
func (a *App) Context(env string) (string, error) {
	e, err := a.envObject(env)
	if err != nil {
		return "", err
	}
	return e.Context, nil
}
func (a *App) BaseProperties() map[string]interface{} {
	p := a.inner.Spec.BaseProperties
	if p == nil {
		return map[string]interface{}{}
	}
	return p
}
func deepMerge(base, overrides map[string]interface{}) map[string]interface{} {
	ret := map[string]interface{}{}
	for k, v := range base {
		ret[k] = v
	}
	for k := range overrides {
		v1, present := base[k]
		v2 := overrides[k]
		if !present {
			ret[k] = v2
			continue
		}
		v1Map, ok1 := v1.(map[string]interface{})
		v2Map, ok2 := v2.(map[string]interface{})
		if ok1 && ok2 {
			ret[k] = deepMerge(v1Map, v2Map)
			continue
		}
		ret[k] = v2
	}
	return ret
}
func (a *App) Properties(env string) (map[string]interface{}, error) {
	if env == Baseline {
		return a.BaseProperties(), nil
	}
	e, err := a.envObject(env)
	if err != nil {
		return nil, err
	}
	eProps := e.Properties
	if e.Properties == nil {
		eProps = map[string]interface{}{}
	}
	return deepMerge(a.BaseProperties(), eProps), nil
}
func (a *App) DefaultNamespace(env string) string {
	var ns string
	if a.overrideNs != "" {
		ns = a.overrideNs
	} else {
		envObj, ok := a.inner.Spec.Environments[env]
		if ok {
			ns = envObj.DefaultNamespace
		}
		if ns == "" {
			ns = "default"
		}
	}
	if a.tag != "" && a.inner.Spec.NamespaceTagSuffix {
		ns += "-" + a.tag
	}
	return ns
}
func (a *App) ComponentsForEnvironment(env string, includes, excludes []string) ([]Component, error) {
	toList := func(m map[string]Component) []Component {
		var ret []Component
		for _, v := range m {
			ret = append(ret, v)
		}
		sort.Slice(ret, func(i, j int) bool {
			return ret[i].Name < ret[j].Name
		})
		return ret
	}
	cf, err := NewComponentFilter(includes, excludes)
	if err != nil {
		return nil, err
	}
	if err := a.verifyComponentList("specified components", includes); err != nil {
		return nil, err
	}
	if err := a.verifyComponentList("specified components", excludes); err != nil {
		return nil, err
	}
	ret := map[string]Component{}
	if env == Baseline {
		for k, v := range a.defaultComponents {
			ret[k] = v
		}
	} else {
		e, err := a.envObject(env)
		if err != nil {
			return nil, err
		}
		for k, v := range a.defaultComponents {
			ret[k] = v
		}
		for _, k := range e.Excludes {
			if _, ok := ret[k]; !ok {
				sio.Warnf("component %s excluded from %s is already excluded by default\n", k, env)
			}
			delete(ret, k)
		}
		for _, k := range e.Includes {
			if _, ok := ret[k]; ok {
				sio.Warnf("component %s included from %s is already included by default\n", k, env)
			}
			ret[k] = a.allComponents[k]
		}
	}
	if !cf.HasFilters() {
		return toList(ret), nil
	}
	for _, k := range includes {
		if _, ok := ret[k]; !ok {
			sio.Noticef("not including component %s since it is not part of the component list for %s\n", k, env)
		}
	}
	subret := map[string]Component{}
	for k, v := range ret {
		if cf.ShouldInclude(v.Name) {
			subret[k] = v
		}
	}
	return toList(subret), nil
}
func (a *App) Environments() map[string]Environment {
	return a.inner.Spec.Environments
}
func (a *App) DeclaredVars() map[string]interface{} {
	ret := map[string]interface{}{}
	for _, v := range a.inner.Spec.Vars.External {
		ret[v.Name] = v.Default
	}
	return ret
}
func (a *App) DeclaredTopLevelVars() map[string]interface{} {
	ret := map[string]interface{}{}
	for _, v := range a.inner.Spec.Vars.TopLevel {
		ret[v.Name] = true
	}
	return ret
}
func (a *App) DeclaredComputedVars() []ComputedVar {
	return a.inner.Spec.Vars.Computed
}
func (a *App) DataSources() []string {
	return a.inner.Spec.DataSources
}
func (a *App) DataSourceExamples() map[string]interface{} {
	ret := a.inner.Spec.DataSourceExamples
	if ret == nil {
		ret = map[string]interface{}{}
	}
	return ret
}