<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const Name = "mantl-api"
const Version = "0.2.2"

var wg sync.WaitGroup

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().String("consul-acl-token", "", "Consul ACL token for accessing mantl-install/apps path")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret", "", "Deprecated. Use mesos-secret-path instead")
	rootCmd.PersistentFlags().String("mesos-secret-path", "/etc/sysconfig/mantl-api", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")
	rootCmd.PersistentFlags().Int("consul-refresh-interval", 10, "The number of seconds after which to check consul for package requests")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			syncRepo(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonHosts := NewDiscovery(client, "marathon", "").discoveredHosts
		if len(marathonHosts) > 0 {
			marathonUrl = fmt.Sprintf("http://%s", marathonHosts[0])
		} else {
			marathonUrl = "http://localhost:8080"
		}<commit_msg>first attempt at vault integration<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	vault "github.com/hashicorp/vault/api"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const Name = "mantl-api"
const Version = "0.2.2"

var wg sync.WaitGroup

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().String("consul-acl-token", "", "Consul ACL token for accessing mantl-install/apps path")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret", "", "Deprecated. Use mesos-secret-path instead")
	rootCmd.PersistentFlags().String("mesos-secret-path", "/etc/sysconfig/mantl-api", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")
	rootCmd.PersistentFlags().Int("consul-refresh-interval", 10, "The number of seconds after which to check consul for package requests")
	rootCmd.PersistentFlags().String("vault-cubbyhole-token", "", "token for retrieving secrets from vault")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			syncRepo(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	if wrapped := viper.GetString("vault-cubbyhole-token"); wrapped != "" {
		config := vault.DefaultConfig()
		err := config.ReadEnvironment()
		if err != nil {
			log.WithError(err).Fatal("Error reading environment for Vault configuration")
		}

		client, err := vault.NewClient(config)
		if err != nil {
			log.WithError(err).Fatal("Error initializing Vault client")
		}

		token, err := client.Logical().Unwrap(wrapped)
		if err != nil {
			log.WithError(err).Fatal("Error unwrapping token")
		} else if token.WrapInfo != nil {
			log.Fatal("Secret appears to be doubly wrapped")
		} else if token.Auth == nil {
			log.Fatal("Secret contained no auth data")
		}

		client.SetToken(token.Auth.ClientToken)

		secret, err := client.Logical().Read("secret/mantl-api")
		if err != nil {
			log.WithError(err).Fatal("Error reading secret/mantl-api")
		}

		for _, secretName := range []string{
			"mesos-principal", "mesos-secret",
			"marathon-user", "marathon-password",
		} {
			secretValue, ok := secret.Data[secretName].(string)
			if ok {
				viper.Set(secretName, secretValue)
			} else {
				log.Warnf("secret/mantl-api didn't contain %s", secretName)
			}
		}
	}

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonHosts := NewDiscovery(client, "marathon", "").discoveredHosts
		if len(marathonHosts) > 0 {
			marathonUrl = fmt.Sprintf("http://%s", marathonHosts[0])
		} else {
			marathonUrl = "http://localhost:8080"
		}
{'fraction_non_alphanumeric': 0.16292502179598953, 'fraction_numerical': 0.0054489973844812556, 'mean_word_length': 5.013761467889908, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '427332', 'n_tokens_mistral': 3359, 'n_tokens_neox': 3148, 'n_words': 645}