<commit_before>package config

import (
	"bytes"
	"errors"
	"fmt"
	"strconv"
	"strings"
)

// StoredRequests configures the backend used to store requests on the server.
type StoredRequests struct {
	// Files should be true if Stored Requests should be loaded from the filesystem.
	Files bool `mapstructure:"filesystem"`
	// Postgres should be non-nil if Stored Requests should be loaded from a Postgres database.
	Postgres *PostgresConfig `mapstructure:"postgres"`
	// Cache should be non-nil if an in-memory cache should be used to store Stored Requests locally.
	InMemoryCache *InMemoryCache `mapstructure:"in_memory_cache"`
}

func (cfg *StoredRequests) validate() error {
	if cfg.Files && cfg.Postgres != nil {
		return errors.New("Only one backend from {filesystem, postgres} can be used at the same time.")
	}

	return nil
}

// PostgresConfig configures the Postgres connection for Stored Requests
type PostgresConfig struct {
	Database string `mapstructure:"dbname"`
	Host     string `mapstructure:"host"`
	Port     int    `mapstructure:"port"`
	Username string `mapstructure:"user"`
	Password string `mapstructure:"password"`

	// QueryTemplate is the Postgres Query which can be used to fetch configs from the database.
	// It is a Template, rather than a full Query, because a single HTTP request may reference multiple Stored Requests.
	//
	// In the simplest case, this could be something like:
	//   SELECT id, requestData FROM stored_requests WHERE id in %ID_LIST%
	//
	// The MakeQuery function will transform this query into:
	//   SELECT id, requestData FROM stored_requests WHERE id in ($1, $2, $3, ...)
	//
	// ... where the number of "$x" args depends on how many IDs are nested within the HTTP request.
	QueryTemplate string `mapstructure:"query"`
}
<commit_msg>Improve error message for bad configs (#313)<commit_after>package config

import (
	"bytes"
	"errors"
	"fmt"
	"strconv"
	"strings"
)

// StoredRequests configures the backend used to store requests on the server.
type StoredRequests struct {
	// Files should be true if Stored Requests should be loaded from the filesystem.
	Files bool `mapstructure:"filesystem"`
	// Postgres should be non-nil if Stored Requests should be loaded from a Postgres database.
	Postgres *PostgresConfig `mapstructure:"postgres"`
	// Cache should be non-nil if an in-memory cache should be used to store Stored Requests locally.
	InMemoryCache *InMemoryCache `mapstructure:"in_memory_cache"`
}

func (cfg *StoredRequests) validate() error {
	if cfg.Files && cfg.Postgres != nil {
		return errors.New("stored request backend is ambiguous. If stored_requests.postgres is defined, then stored_requests.filesystem must be false")
	}

	return nil
}

// PostgresConfig configures the Postgres connection for Stored Requests
type PostgresConfig struct {
	Database string `mapstructure:"dbname"`
	Host     string `mapstructure:"host"`
	Port     int    `mapstructure:"port"`
	Username string `mapstructure:"user"`
	Password string `mapstructure:"password"`

	// QueryTemplate is the Postgres Query which can be used to fetch configs from the database.
	// It is a Template, rather than a full Query, because a single HTTP request may reference multiple Stored Requests.
	//
	// In the simplest case, this could be something like:
	//   SELECT id, requestData FROM stored_requests WHERE id in %ID_LIST%
	//
	// The MakeQuery function will transform this query into:
	//   SELECT id, requestData FROM stored_requests WHERE id in ($1, $2, $3, ...)
	//
	// ... where the number of "$x" args depends on how many IDs are nested within the HTTP request.
	QueryTemplate string `mapstructure:"query"`
}

{'fraction_non_alphanumeric': 0.08632502055357633, 'fraction_numerical': 0.002466429158673609, 'mean_word_length': 4.169971671388102, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '86957', 'n_tokens_mistral': 1031, 'n_tokens_neox': 1012, 'n_words': 439}