<commit_before>package server

import (
	"github.com/Bogh/gcm"
	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"

	"fmt"
	"net"
	"runtime"
	"strconv"
	"strings"
)

const (
	defaultHttpListen      = ":8080"
	defaultHealthEndpoint  = "/admin/healthcheck"
	defaultMetricsEndpoint = "/admin/metrics"
	defaultKVSBackend      = "file"
	defaultMSBackend       = "file"
	defaultStoragePath     = "/var/lib/guble"
	defaultNodePort        = "10000"
	development            = "dev"
	integration            = "int"
	preproduction          = "pre"
	production             = "prod"
	memProfile             = "mem"
	cpuProfile             = "cpu"
	blockProfile           = "block"
)

var (
	defaultGCMEndpoint = gcm.GcmSendEndpoint
	environments       = []string{development, integration, preproduction, production}
)

type (
	// PostgresConfig is used for configuring the Postgresql connection.
	PostgresConfig struct {
		Host     *string
		Port     *int
		User     *string
		Password *string
		DbName   *string
	}
	// GCMConfig is used for configuring the Google Cloud Messaging component.
	GCMConfig struct {
		Enabled  *bool
		APIKey   *string
		Workers  *int
		Endpoint *string
	}
	// ClusterConfig is used for configuring the cluster component.
	ClusterConfig struct {
		NodeID   *int
		NodePort *int
		Remotes  *tcpAddrList
	}
	// Config is used for configuring Guble (including its components).
	Config struct {
		Log             *string
		EnvName         *string
		HttpListen      *string
		KVS             *string
		MS              *string
		StoragePath     *string
		HealthEndpoint  *string
		MetricsEndpoint *string
		Profile         *string
		Postgres        PostgresConfig
		GCM             GCMConfig
		Cluster         ClusterConfig
	}
)

var (
	parsed = false
	config = &Config{
		Log: kingpin.Flag("log", "Log level").
			Default(log.ErrorLevel.String()).
			Envar("GUBLE_LOG").
			Enum(logLevels()...),
		EnvName: kingpin.Flag("env", `Name of the environment on which the application is running`).
			Default(development).
			Envar("GUBLE_ENV").
			Enum(environments...),
		HttpListen: kingpin.Flag("http", `The address to for the HTTP server to listen on (format: "[Host]:Port")`).
			Default(defaultHttpListen).
			Envar("GUBLE_HTTP_LISTEN").
			String(),
		KVS: kingpin.Flag("kvs", "The storage backend for the key-value store to use : file | memory").
			Default(defaultKVSBackend).
			HintOptions("file", "memory").
			Envar("GUBLE_KVS").
			String(),
		MS: kingpin.Flag("ms", "The message storage backend : file | memory").
			Default(defaultMSBackend).
			HintOptions("file", "memory").
			Envar("GUBLE_MS").
			String(),
		StoragePath: kingpin.Flag("storage-path", "The path for storing messages and key-value data if 'file' is selected").
			Default(defaultStoragePath).
			Envar("GUBLE_STORAGE_PATH").
			ExistingDir(),
		HealthEndpoint: kingpin.Flag("health-endpoint", `The health endpoint to be used by the HTTP server (value for disabling it: "")`).
			Default(defaultHealthEndpoint).
			Envar("GUBLE_HEALTH_ENDPOINT").
			String(),
		MetricsEndpoint: kingpin.Flag("metrics-endpoint", `The metrics endpoint to be used by the HTTP server (value for disabling it: "")`).
			Default(defaultMetricsEndpoint).
			Envar("GUBLE_METRICS_ENDPOINT").
			String(),
		Profile: kingpin.Flag("profile", `The profiler to be used (default: none)`).
			Default("").
			HintOptions("mem", "cpu", "block").
			Envar("GUBLE_PROFILE").
			Enum("mem", "cpu", "block", ""),
		Postgres: PostgresConfig{
			Host: kingpin.Flag("pg-host", "The PostgreSQL hostname").
				Default("localhost").
				Envar("GUBLE_PG_HOST").
				String(),
			Port: kingpin.Flag("pg-port", "The PostgreSQL port").
				Default("5432").
				Envar("GUBLE_PG_PORT").
				Int(),
			User: kingpin.Flag("pg-user", "The PostgreSQL user").
				Default("guble").
				Envar("GUBLE_PG_USER").
				String(),
			Password: kingpin.Flag("pg-password", "The PostgreSQL password").
				Default("guble").
				Envar("GUBLE_PG_PASSWORD").
				String(),
			DbName: kingpin.Flag("pg-dbname", "The PostgreSQL database name").
				Default("guble").
				Envar("GUBLE_PG_DBNAME").
				String(),
		},
		GCM: GCMConfig{
			Enabled: kingpin.Flag("gcm", "Enable the Google Cloud Messaging Connector").
				Envar("GUBLE_GCM").
				Bool(),
			APIKey: kingpin.Flag("gcm-api-key", "The Google API Key for Google Cloud Messaging").
				Envar("GUBLE_GCM_API_KEY").
				String(),
			Workers: kingpin.Flag("gcm-workers", "The number of workers handling traffic with Google Cloud Messaging (default: GOMAXPROCS)").
				Default(strconv.Itoa(runtime.GOMAXPROCS(0))).
				Envar("GUBLE_GCM_WORKERS").
				Int(),
			Endpoint: kingpin.Flag("gcm-endpoint", "The Google Cloud Messaging endpoint").
				Default(defaultGCMEndpoint).
				Envar("GUBLE_GCM_ENDPOINT").
				String(),
		},
		Cluster: ClusterConfig{
			NodeID: kingpin.Flag("node-id", "(cluster mode) This guble node's own ID: a strictly positive integer number which must be unique in cluster").
				Envar("GUBLE_NODE_ID").
				Int(),
			NodePort: kingpin.Flag("node-port", "(cluster mode) This guble node's own local port: a strictly positive integer number").
				Default(defaultNodePort).
				Envar("GUBLE_NODE_PORT").
				Int(),
			Remotes: stringToTcpAddrList(kingpin.Flag("remotes", `(cluster mode) The list of TCP addresses of some other guble nodes (format: "IP:port")`).
				Envar("GUBLE_NODE_REMOTES")),
		},
	}
)

func logLevels() (levels []string) {
	for _, level := range log.AllLevels {
		levels = append(levels, level.String())
	}
	return
}

// parseConfig parses the flags from command line. Must be used before accessing the config.
// If there are missing or invalid arguments it will exit the application
// and display a message.
func parseConfig() {
	if parsed {
		return
	}
	kingpin.Parse()
	parsed = true
	return
}

type tcpAddrList []*net.TCPAddr

func (h *tcpAddrList) Set(value string) error {
	addresses := strings.Split(value, " ")

	// Recreate the list on `Set`
	*h = make(tcpAddrList, 0)
	for _, addr := range addresses {
		logger.WithField("addr", addr).Info("value")
		parts := strings.SplitN(addr, ":", 2)
		if len(parts) != 2 {
			return fmt.Errorf("expected HEADER:VALUE got '%s'", addr)
		}
		addr, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return err
		}
		*h = append(*h, addr)
	}
	return nil
}

func stringToTcpAddrList(s kingpin.Settings) *tcpAddrList {
	slist := make(tcpAddrList, 0)
	s.SetValue(&slist)
	return &slist
}

func (h *tcpAddrList) String() string {
	return ""
}
<commit_msg>minor improvements in config Flags and docs<commit_after>package server

import (
	"github.com/Bogh/gcm"
	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"

	"fmt"
	"net"
	"runtime"
	"strconv"
	"strings"
)

const (
	defaultHttpListen      = ":8080"
	defaultHealthEndpoint  = "/admin/healthcheck"
	defaultMetricsEndpoint = "/admin/metrics"
	defaultKVSBackend      = "file"
	defaultMSBackend       = "file"
	defaultStoragePath     = "/var/lib/guble"
	defaultNodePort        = "10000"
	development            = "dev"
	integration            = "int"
	preproduction          = "pre"
	production             = "prod"
	memProfile             = "mem"
	cpuProfile             = "cpu"
	blockProfile           = "block"
)

var (
	defaultGCMEndpoint = gcm.GcmSendEndpoint
	environments       = []string{development, integration, preproduction, production}
)

type (
	// PostgresConfig is used for configuring the Postgresql connection.
	PostgresConfig struct {
		Host     *string
		Port     *int
		User     *string
		Password *string
		DbName   *string
	}
	// GCMConfig is used for configuring the Google Cloud Messaging component.
	GCMConfig struct {
		Enabled  *bool
		APIKey   *string
		Workers  *int
		Endpoint *string
	}
	// ClusterConfig is used for configuring the cluster component.
	ClusterConfig struct {
		NodeID   *int
		NodePort *int
		Remotes  *tcpAddrList
	}
	// Config is used for configuring Guble (including its components).
	Config struct {
		Log             *string
		EnvName         *string
		HttpListen      *string
		KVS             *string
		MS              *string
		StoragePath     *string
		HealthEndpoint  *string
		MetricsEndpoint *string
		Profile         *string
		Postgres        PostgresConfig
		GCM             GCMConfig
		Cluster         ClusterConfig
	}
)

var (
	parsed = false
	config = &Config{
		Log: kingpin.Flag("log", "Log level").
			Default(log.ErrorLevel.String()).
			Envar("GUBLE_LOG").
			Enum(logLevels()...),
		EnvName: kingpin.Flag("env", `Name of the environment on which the application is running`).
			Default(development).
			Envar("GUBLE_ENV").
			Enum(environments...),
		HttpListen: kingpin.Flag("http", `The address to for the HTTP server to listen on (format: "[Host]:Port")`).
			Default(defaultHttpListen).
			Envar("GUBLE_HTTP_LISTEN").
			String(),
		KVS: kingpin.Flag("kvs", "The storage backend for the key-value store to use : file | memory").
			Default(defaultKVSBackend).
			Envar("GUBLE_KVS").
			String(),
		MS: kingpin.Flag("ms", "The message storage backend : file | memory").
			Default(defaultMSBackend).
			HintOptions("file", "memory").
			Envar("GUBLE_MS").
			String(),
		StoragePath: kingpin.Flag("storage-path", "The path for storing messages and key-value data if 'file' is selected").
			Default(defaultStoragePath).
			Envar("GUBLE_STORAGE_PATH").
			ExistingDir(),
		HealthEndpoint: kingpin.Flag("health-endpoint", `The health endpoint to be used by the HTTP server (value for disabling it: "")`).
			Default(defaultHealthEndpoint).
			Envar("GUBLE_HEALTH_ENDPOINT").
			String(),
		MetricsEndpoint: kingpin.Flag("metrics-endpoint", `The metrics endpoint to be used by the HTTP server (value for disabling it: "")`).
			Default(defaultMetricsEndpoint).
			Envar("GUBLE_METRICS_ENDPOINT").
			String(),
		Profile: kingpin.Flag("profile", `The profiler to be used (default: none): mem | cpu | block`).
			Default("").
			Envar("GUBLE_PROFILE").
			Enum("mem", "cpu", "block", ""),
		Postgres: PostgresConfig{
			Host: kingpin.Flag("pg-host", "The PostgreSQL hostname").
				Default("localhost").
				Envar("GUBLE_PG_HOST").
				String(),
			Port: kingpin.Flag("pg-port", "The PostgreSQL port").
				Default("5432").
				Envar("GUBLE_PG_PORT").
				Int(),
			User: kingpin.Flag("pg-user", "The PostgreSQL user").
				Default("guble").
				Envar("GUBLE_PG_USER").
				String(),
			Password: kingpin.Flag("pg-password", "The PostgreSQL password").
				Default("<PASSWORD>").
				Envar("GUBLE_PG_PASSWORD").
				String(),
			DbName: kingpin.Flag("pg-dbname", "The PostgreSQL database name").
				Default("guble").
				Envar("GUBLE_PG_DBNAME").
				String(),
		},
		GCM: GCMConfig{
			Enabled: kingpin.Flag("gcm", "Enable the Google Cloud Messaging Connector").
				Envar("GUBLE_GCM").
				Bool(),
			APIKey: kingpin.Flag("gcm-api-key", "The Google API Key for Google Cloud Messaging").
				Envar("GUBLE_GCM_API_KEY").
				String(),
			Workers: kingpin.Flag("gcm-workers", "The number of workers handling traffic with Google Cloud Messaging (default: GOMAXPROCS)").
				Default(strconv.Itoa(runtime.GOMAXPROCS(0))).
				Envar("GUBLE_GCM_WORKERS").
				Int(),
			Endpoint: kingpin.Flag("gcm-endpoint", "The Google Cloud Messaging endpoint").
				Default(defaultGCMEndpoint).
				Envar("GUBLE_GCM_ENDPOINT").
				String(),
		},
		Cluster: ClusterConfig{
			NodeID: kingpin.Flag("node-id", "(cluster mode) This guble node's own ID: a strictly positive integer number which must be unique in cluster").
				Envar("GUBLE_NODE_ID").
				Int(),
			NodePort: kingpin.Flag("node-port", "(cluster mode) This guble node's own local port: a strictly positive integer number").
				Default(defaultNodePort).
				Envar("GUBLE_NODE_PORT").
				Int(),
			Remotes: stringToTcpAddrList(kingpin.Flag("remotes", `(cluster mode) The list of TCP addresses of some other guble nodes (format: "IP:port")`).
				Envar("GUBLE_NODE_REMOTES")),
		},
	}
)

func logLevels() (levels []string) {
	for _, level := range log.AllLevels {
		levels = append(levels, level.String())
	}
	return
}

// parseConfig parses the flags from command line. Must be used before accessing the config.
// If there are missing or invalid arguments it will exit the application
// and display a message.
func parseConfig() {
	if parsed {
		return
	}
	kingpin.Parse()
	parsed = true
	return
}

type tcpAddrList []*net.TCPAddr

func (h *tcpAddrList) Set(value string) error {
	addresses := strings.Split(value, " ")

	// Recreate the list on `Set`
	*h = make(tcpAddrList, 0)
	for _, addr := range addresses {
		logger.WithField("addr", addr).Info("value")
		parts := strings.SplitN(addr, ":", 2)
		if len(parts) != 2 {
			return fmt.Errorf("expected HEADER:VALUE got '%s'", addr)
		}
		addr, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return err
		}
		*h = append(*h, addr)
	}
	return nil
}

func stringToTcpAddrList(s kingpin.Settings) *tcpAddrList {
	slist := make(tcpAddrList, 0)
	s.SetValue(&slist)
	return &slist
}

func (h *tcpAddrList) String() string {
	return ""
}

{'fraction_non_alphanumeric': 0.13224152445090687, 'fraction_numerical': 0.00290808907936022, 'mean_word_length': 3.479945149125814, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4354817', 'n_tokens_mistral': 4732, 'n_tokens_neox': 4202, 'n_words': 1122}