<commit_before>/*
Copyright 2012 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serverinit

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/jsonsign"
	"camlistore.org/pkg/osutil"
	"camlistore.org/pkg/sorted"
	"camlistore.org/pkg/types/serverconfig"
	"go4.org/jsonconfig"

	"go4.org/wkfs"
)

var (
	tempDir = os.TempDir
	noMkdir bool // for tests to not call os.Mkdir
)

type tlsOpts struct {
	httpsCert string
	httpsKey  string
}

// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) {
	b := &lowBuilder{
		high: conf,
		low: jsonconfig.Obj{
			"prefixes": make(map[string]interface{}),
		},
	}
	return b.build()
}

// A lowBuilder builds a low-level config from a high-level config.
type lowBuilder struct {
	high *serverconfig.Config // high-level config (input)
	low  jsonconfig.Obj       // low-level handler config (output)
}

// args is an alias for map[string]interface{} just to cut down on
// noise below.  But we take care to convert it back to
// map[string]interface{} in the one place where we accept it.
type args map[string]interface{}

func (b *lowBuilder) addPrefix(at, handler string, a args) {
	v := map[string]interface{}{
		"handler": handler,
	}
	if a != nil {
		v["handlerArgs"] = (map[string]interface{})(a)
	}
	b.low["prefixes"].(map[string]interface{})[at] = v
}

func (b *lowBuilder) hasPrefix(p string) bool {
	_, ok := b.low["prefixes"].(map[string]interface{})[p]
	return ok
}

func (b *lowBuilder) runIndex() bool          { return b.high.RunIndex.Get() }
func (b *lowBuilder) copyIndexToMemory() bool { return b.high.CopyIndexToMemory.Get() }

// dbName returns which database to use for the provided user ("of").
// The user should be a key as described in pkg/types/serverconfig/config.go's
// description of DBNames: "index", "queue-sync-to-index", etc.
func (b *lowBuilder) dbName(of string) string {
	if v, ok := b.high.DBNames[of]; ok && v != "" {
		return v
	}
	if of == "index" {
		if b.high.DBName != "" {
			return b.high.DBName
		}
		username := osutil.Username()
		if username == "" {
			envVar := "USER"
			if runtime.GOOS == "windows" {
				envVar += "NAME"
			}
			return "camlistore_index"
		}
		return "camli" + username
	}
	return ""
}

var errNoOwner = errors.New("no owner")

// Error is errNoOwner if no identity configured
func (b *lowBuilder) searchOwner() (br blob.Ref, err error) {
	if b.high.Identity == "" {
		return br, errNoOwner
	}
	entity, err := jsonsign.EntityFromSecring(b.high.Identity, b.high.IdentitySecretRing)
	if err != nil {
		return br, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return br, err
	}
	return blob.SHA1FromString(armoredPublicKey), nil
}

func (b *lowBuilder) addPublishedConfig(tlsO *tlsOpts) error {
	published := b.high.Publish
	for k, v := range published {
		if v.CamliRoot == "" {
			return fmt.Errorf("Missing \"camliRoot\" key in configuration for %s.", k)
		}
		if v.GoTemplate == "" {
			return fmt.Errorf("Missing \"goTemplate\" key in configuration for %s.", k)
		}

		appConfig := map[string]interface{}{
			"camliRoot":  v.CamliRoot,
			"cacheRoot":  v.CacheRoot,
			"goTemplate": v.GoTemplate,
		}
		if v.HTTPSCert != "" && v.HTTPSKey != "" {
			// user can specify these directly in the publish section
			appConfig["httpsCert"] = v.HTTPSCert
			appConfig["httpsKey"] = v.HTTPSKey
		} else {
			// default to Camlistore parameters, if any
			if tlsO != nil {
				appConfig["httpsCert"] = tlsO.httpsCert
				appConfig["httpsKey"] = tlsO.httpsKey
			}
		}
		a := args{
			"program":   v.Program,
			"appConfig": appConfig,
		}
		if v.BaseURL != "" {
			a["baseURL"] = v.BaseURL
		}
		program := "publisher"
		if v.Program != "" {
			program = v.Program
		}
		a["program"] = program
		b.addPrefix(k, "app", a)
	}
	return nil
}

// kvFileType returns the file based sorted type defined for index storage, if
// any. It defaults to "leveldb" otherwise.
func (b *lowBuilder) kvFileType() string {
	switch {
	case b.high.SQLite != "":
		return "sqlite"
	case b.high.KVFile != "":
		return "kv"
	case b.high.LevelDB != "":
		return "leveldb"
	default:
		return sorted.DefaultKVFileType
	}
}

func (b *lowBuilder) addUIConfig() {
	args := map[string]interface{}{
		"cache": "/cache/",
	}
	if b.high.SourceRoot != "" {
		args["sourceRoot"] = b.high.SourceRoot
	}
	var thumbCache map[string]interface{}
	if b.high.BlobPath != "" {
		thumbCache = map[string]interface{}{
			"type": b.kvFileType(),
			"file": filepath.Join(b.high.BlobPath, "thumbmeta."+b.kvFileType()),
		}
	}
	if thumbCache == nil {
		sorted, err := b.sortedStorage("ui_thumbcache")
		if err == nil {
			thumbCache = sorted
		}
	}
	if thumbCache != nil {
		args["scaledImage"] = thumbCache
	}
	b.addPrefix("/ui/", "ui", args)
}

func (b *lowBuilder) mongoIndexStorage(confStr, sortedType string) (map[string]interface{}, error) {
	dbName := b.dbName(sortedType)
	if dbName == "" {
		return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
	}
	fields := strings.Split(confStr, "@")
	if len(fields) == 2 {
		host := fields[1]
		fields = strings.Split(fields[0], ":")
		if len(fields) == 2 {
			user, pass := fields[0], fields[1]
			return map[string]interface{}{
				"type":     "mongo",
				"host":     host,
				"user":     user,
				"password": <PASSWORD>,
				"database": dbName,
			}, nil
		}
	}
	return nil, errors.New("Malformed mongo config string; want form: \"user:password@host\"")
}

// parses "user@host:password", which you think would be easy, but we
// documented this format without thinking about port numbers, so this
// uses heuristics to guess what extra colons mean.
func parseUserHostPass(v string) (user, host, password string, ok bool) {
	f := strings.SplitN(v, "@", 2)
	if len(f) != 2 {
		return
	}
	user = f[0]
	f = strings.Split(f[1], ":")
	if len(f) < 2 {
		return "", "", "", false
	}
	host = f[0]
	f = f[1:]
	if len(f) >= 2 {
		if _, err := strconv.ParseUint(f[0], 10, 16); err == nil {
			host = host + ":" + f[0]
			f = f[1:]
		}
	}
	password = strings.Join(f, ":")
	ok = true
	return
}

func (b *lowBuilder) dbIndexStorage(rdbms string, confStr string, sortedType string) (map[string]interface{}, error) {
	dbName := b.dbName(sortedType)
	if dbName == "" {
		return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
	}
	user, host, password, ok := parseUserHostPass(confStr)
	if !ok {
		return nil, fmt.Errorf("Malformed %s config string. Want: \"user@host:password\"", rdbms)
	}
	return map[string]interface{}{
		"type":     rdbms,
		"host":     host,
		"user":     user,
		"password": password,
		"database": b.dbName(sortedType),
	}, nil
}

func (b *lowBuilder) sortedStorage(sortedType string) (map[string]interface{}, error) {
	return b.sortedStorageAt(sortedType, "")
}

// filePrefix gives a file path of where to put the database. It can be omitted by
// some sorted implementations, but is required by others.
// The filePrefix should be to a file, not a directory, and should not end in a ".ext" extension.
// An extension like ".kv" or ".sqlite" will be added.
func (b *lowBuilder) sortedStorageAt(sortedType, filePrefix string) (map[string]interface{}, error) {
	if b.high.MySQL != "" {
		return b.dbIndexStorage("mysql", b.high.MySQL, sortedType)
	}
	if b.high.PostgreSQL != "" {
		return b.dbIndexStorage("postgres", b.high.PostgreSQL, sortedType)
	}
	if b.high.Mongo != "" {
		return b.mongoIndexStorage(b.high.Mongo, sortedType)
	}
	if b.high.MemoryIndex {
		return map[string]interface{}{
			"type": "memory",
		}, nil
	}
	if sortedType != "index" && filePrefix == "" {
		return nil, fmt.Errorf("internal error: use of sortedStorageAt with a non-index type and no file location for non-database sorted implementation")
	}
	// dbFile returns path directly if sortedType == "index", else it returns filePrefix+"."+ext.
	dbFile := func(path, ext string) string {
		if sortedType == "index" {
			return path
		}
		return filePrefix + "." + ext
	}<commit_msg>pkg/serverinit: improve genconfig error message<commit_after>/*
Copyright 2012 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serverinit

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/jsonsign"
	"camlistore.org/pkg/osutil"
	"camlistore.org/pkg/sorted"
	"camlistore.org/pkg/types/serverconfig"
	"go4.org/jsonconfig"

	"go4.org/wkfs"
)

var (
	tempDir = os.TempDir
	noMkdir bool // for tests to not call os.Mkdir
)

type tlsOpts struct {
	httpsCert string
	httpsKey  string
}

// genLowLevelConfig returns a low-level config from a high-level config.
func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err error) {
	b := &lowBuilder{
		high: conf,
		low: jsonconfig.Obj{
			"prefixes": make(map[string]interface{}),
		},
	}
	return b.build()
}

// A lowBuilder builds a low-level config from a high-level config.
type lowBuilder struct {
	high *serverconfig.Config // high-level config (input)
	low  jsonconfig.Obj       // low-level handler config (output)
}

// args is an alias for map[string]interface{} just to cut down on
// noise below.  But we take care to convert it back to
// map[string]interface{} in the one place where we accept it.
type args map[string]interface{}

func (b *lowBuilder) addPrefix(at, handler string, a args) {
	v := map[string]interface{}{
		"handler": handler,
	}
	if a != nil {
		v["handlerArgs"] = (map[string]interface{})(a)
	}
	b.low["prefixes"].(map[string]interface{})[at] = v
}

func (b *lowBuilder) hasPrefix(p string) bool {
	_, ok := b.low["prefixes"].(map[string]interface{})[p]
	return ok
}

func (b *lowBuilder) runIndex() bool          { return b.high.RunIndex.Get() }
func (b *lowBuilder) copyIndexToMemory() bool { return b.high.CopyIndexToMemory.Get() }

// dbName returns which database to use for the provided user ("of").
// The user should be a key as described in pkg/types/serverconfig/config.go's
// description of DBNames: "index", "queue-sync-to-index", etc.
func (b *lowBuilder) dbName(of string) string {
	if v, ok := b.high.DBNames[of]; ok && v != "" {
		return v
	}
	if of == "index" {
		if b.high.DBName != "" {
			return b.high.DBName
		}
		username := osutil.Username()
		if username == "" {
			envVar := "USER"
			if runtime.GOOS == "windows" {
				envVar += "NAME"
			}
			return "camlistore_index"
		}
		return "camli" + username
	}
	return ""
}

var errNoOwner = errors.New("no owner")

// Error is errNoOwner if no identity configured
func (b *lowBuilder) searchOwner() (br blob.Ref, err error) {
	if b.high.Identity == "" {
		return br, errNoOwner
	}
	entity, err := jsonsign.EntityFromSecring(b.high.Identity, b.high.IdentitySecretRing)
	if err != nil {
		return br, err
	}
	armoredPublicKey, err := jsonsign.ArmoredPublicKey(entity)
	if err != nil {
		return br, err
	}
	return blob.SHA1FromString(armoredPublicKey), nil
}

func (b *lowBuilder) addPublishedConfig(tlsO *tlsOpts) error {
	published := b.high.Publish
	for k, v := range published {
		if v.CamliRoot == "" {
			return fmt.Errorf("Missing \"camliRoot\" key in configuration for %s.", k)
		}
		if v.GoTemplate == "" {
			return fmt.Errorf("Missing \"goTemplate\" key in configuration for %s.", k)
		}

		appConfig := map[string]interface{}{
			"camliRoot":  v.CamliRoot,
			"cacheRoot":  v.CacheRoot,
			"goTemplate": v.GoTemplate,
		}
		if v.HTTPSCert != "" && v.HTTPSKey != "" {
			// user can specify these directly in the publish section
			appConfig["httpsCert"] = v.HTTPSCert
			appConfig["httpsKey"] = v.HTTPSKey
		} else {
			// default to Camlistore parameters, if any
			if tlsO != nil {
				appConfig["httpsCert"] = tlsO.httpsCert
				appConfig["httpsKey"] = tlsO.httpsKey
			}
		}
		a := args{
			"program":   v.Program,
			"appConfig": appConfig,
		}
		if v.BaseURL != "" {
			a["baseURL"] = v.BaseURL
		}
		program := "publisher"
		if v.Program != "" {
			program = v.Program
		}
		a["program"] = program
		b.addPrefix(k, "app", a)
	}
	return nil
}

// kvFileType returns the file based sorted type defined for index storage, if
// any. It defaults to "leveldb" otherwise.
func (b *lowBuilder) kvFileType() string {
	switch {
	case b.high.SQLite != "":
		return "sqlite"
	case b.high.KVFile != "":
		return "kv"
	case b.high.LevelDB != "":
		return "leveldb"
	default:
		return sorted.DefaultKVFileType
	}
}

func (b *lowBuilder) addUIConfig() {
	args := map[string]interface{}{
		"cache": "/cache/",
	}
	if b.high.SourceRoot != "" {
		args["sourceRoot"] = b.high.SourceRoot
	}
	var thumbCache map[string]interface{}
	if b.high.BlobPath != "" {
		thumbCache = map[string]interface{}{
			"type": b.kvFileType(),
			"file": filepath.Join(b.high.BlobPath, "thumbmeta."+b.kvFileType()),
		}
	}
	if thumbCache == nil {
		sorted, err := b.sortedStorage("ui_thumbcache")
		if err == nil {
			thumbCache = sorted
		}
	}
	if thumbCache != nil {
		args["scaledImage"] = thumbCache
	}
	b.addPrefix("/ui/", "ui", args)
}

func (b *lowBuilder) mongoIndexStorage(confStr, sortedType string) (map[string]interface{}, error) {
	dbName := b.dbName(sortedType)
	if dbName == "" {
		return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
	}
	fields := strings.Split(confStr, "@")
	if len(fields) == 2 {
		host := fields[1]
		fields = strings.Split(fields[0], ":")
		if len(fields) == 2 {
			user, pass := fields[0], fields[1]
			return map[string]interface{}{
				"type":     "mongo",
				"host":     host,
				"user":     user,
				"password": <PASSWORD>,
				"database": dbName,
			}, nil
		}
	}
	return nil, errors.New("Malformed mongo config string; want form: \"user:password@host\"")
}

// parses "user@host:password", which you think would be easy, but we
// documented this format without thinking about port numbers, so this
// uses heuristics to guess what extra colons mean.
func parseUserHostPass(v string) (user, host, password string, ok bool) {
	f := strings.SplitN(v, "@", 2)
	if len(f) != 2 {
		return
	}
	user = f[0]
	f = strings.Split(f[1], ":")
	if len(f) < 2 {
		return "", "", "", false
	}
	host = f[0]
	f = f[1:]
	if len(f) >= 2 {
		if _, err := strconv.ParseUint(f[0], 10, 16); err == nil {
			host = host + ":" + f[0]
			f = f[1:]
		}
	}
	password = strings.Join(f, ":")
	ok = true
	return
}

func (b *lowBuilder) dbIndexStorage(rdbms string, confStr string, sortedType string) (map[string]interface{}, error) {
	dbName := b.dbName(sortedType)
	if dbName == "" {
		return nil, fmt.Errorf("no database name configured for sorted store %q", sortedType)
	}
	user, host, password, ok := parseUserHostPass(confStr)
	if !ok {
		return nil, fmt.Errorf("Malformed %s config string. Want: \"user@host:password\"", rdbms)
	}
	return map[string]interface{}{
		"type":     rdbms,
		"host":     host,
		"user":     user,
		"password": password,
		"database": b.dbName(sortedType),
	}, nil
}

func (b *lowBuilder) sortedStorage(sortedType string) (map[string]interface{}, error) {
	return b.sortedStorageAt(sortedType, "")
}

// filePrefix gives a file path of where to put the database. It can be omitted by
// some sorted implementations, but is required by others.
// The filePrefix should be to a file, not a directory, and should not end in a ".ext" extension.
// An extension like ".kv" or ".sqlite" will be added.
func (b *lowBuilder) sortedStorageAt(sortedType, filePrefix string) (map[string]interface{}, error) {
	if b.high.MySQL != "" {
		return b.dbIndexStorage("mysql", b.high.MySQL, sortedType)
	}
	if b.high.PostgreSQL != "" {
		return b.dbIndexStorage("postgres", b.high.PostgreSQL, sortedType)
	}
	if b.high.Mongo != "" {
		return b.mongoIndexStorage(b.high.Mongo, sortedType)
	}
	if b.high.MemoryIndex {
		return map[string]interface{}{
			"type": "memory",
		}, nil
	}
	if sortedType != "index" && filePrefix == "" {
		return nil, fmt.Errorf("internal error: use of sortedStorageAt with a non-index type (%v) and no file location for non-database sorted implementation", sortedType)
	}
	// dbFile returns path directly if sortedType == "index", else it returns filePrefix+"."+ext.
	dbFile := func(path, ext string) string {
		if sortedType == "index" {
			return path
		}
		return filePrefix + "." + ext
	}
{'fraction_non_alphanumeric': 0.1478764257465467, 'fraction_numerical': 0.003668252421619763, 'mean_word_length': 3.702964959568733, 'pattern_counts': {'":': 56, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5802613', 'n_tokens_mistral': 6388, 'n_tokens_neox': 6000, 'n_words': 1746}