<commit_before>package options

import (
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"os"
)

type NodeConf struct {
	Seeds          []string `toml:"seed_servers"`
	UpdateInterval string   `toml:"update_interval"`
}

type Conf struct {
	Root       string   `toml:"root_dir"`
	ApiPort    string   `toml:"api_port"`
	RunNode    bool     `toml:"run_as_node"`
	NodeConfig NodeConf `toml:"node"`
}

var Config Conf

func GetOptions() {
	var configFile string

	flag.StringVar(&configFile, "config", "", "Configuration file")
	flag.StringVar(&Config.Root, "root", "", "Root directory to serve (required)")
	flag.StringVar(&Config.ApiPort, "api-port", "8081", "Port that the API listens on")
	flag.BoolVar(&Config.RunNode, "node", false, "Run as a node")
	flag.StringVar(&Config.NodeConfig.UpdateInterval, "update-interval", "1m", "How often to update with the other servers")

	flag.Parse()

	if configFile != "" {
		if _, err := toml.DecodeFile(configFile, &Config); err != nil {
			fmt.Printf("Error reading config %s: %s\n", configFile, err.Error())
			os.Exit(-1)
		}
		fmt.Printf("Configration file options in %s overriding command line options\n", configFile)
	}

	if Config.Root == "" {
		fmt.Println("Must specify root directory")
		os.Exit(-1)
	}

	if Config.RunNode == true && Config.NodeConfig.Seeds == nil {
		fmt.Println("Must specify seed server when running as node")
		os.Exit(-1)<commit_msg>Updated options.go<commit_after>package options

import (
	"flag"
	"fmt"
	"github.com/BurntSushi/toml"
	"os"
)

type NodeConf struct {
	Seeds          []string `toml:"seed_servers"`
	UpdateInterval string   `toml:"update_interval"`
}

type Conf struct {
	Root       string   `toml:"root_dir"`
	ApiPort    string   `toml:"api_port"`
	RunNode    bool     `toml:"run_as_node"`
	NodeConfig NodeConf `toml:"node"`
}

var Config Conf

func GetOptions() {
	var configFile string

	flag.StringVar(&configFile, "config", "", "Configuration file")
	flag.StringVar(&Config.Root, "root", "", "Root directory to serve (required). Must be absolute path")
	flag.StringVar(&Config.ApiPort, "api-port", "8081", "Port that the API listens on")
	flag.BoolVar(&Config.RunNode, "node", false, "Run as a node")
	flag.StringVar(&Config.NodeConfig.UpdateInterval, "update-interval", "1m", "How often to update with the other servers")

	flag.Parse()

	if configFile != "" {
		if _, err := toml.DecodeFile(configFile, &Config); err != nil {
			fmt.Printf("Error reading config %s: %s\n", configFile, err.Error())
			os.Exit(-1)
		}
		fmt.Printf("Configration file options in %s overriding command line options\n", configFile)
	}

	if Config.Root == "" {
		fmt.Println("Must specify root directory")
		os.Exit(-1)
	}

	if Config.RunNode == true && Config.NodeConfig.Seeds == nil {
		fmt.Println("Must specify seed server when running as node")
		os.Exit(-1)
{'fraction_non_alphanumeric': 0.1489061397318278, 'fraction_numerical': 0.0056457304163726185, 'mean_word_length': 4.230627306273063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4581577', 'n_tokens_mistral': 994, 'n_tokens_neox': 952, 'n_words': 254}