<commit_before>package main

import (
	"admin"
	"api/http"
	"configuration"
	"coordinator"
	"datastore"
	"engine"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
)

const (
	version = "dev"
	gitSha  = ""
)

func main() {
	fileName := flag.String("config", "config.json.sample", "Config file")
	wantsVersion := flag.Bool("v", false, "Get version number")
	pidFile := flag.String("pidfile", "", "the pid file")

	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	if wantsVersion != nil && *wantsVersion {
		fmt.Printf("InfluxDB v%s (git: %s)\n", version, gitSha)
		return
	}
	config := configuration.LoadConfiguration(*fileName)

	if pidFile != nil && *pidFile != "" {
		pid := strconv.Itoa(os.Getpid())
		if err := ioutil.WriteFile(*pidFile, []byte(pid), 0644); err != nil {
			panic(err)
		}
	}

	log.Println("Starting Influx Server...")
	clusterConfig := coordinator.NewClusterConfiguration()
	os.MkdirAll(config.RaftDir, 0744)

	raftServer := coordinator.NewRaftServer(config.RaftDir, "localhost", config.RaftServerPort, clusterConfig)
	go func() {
		raftServer.ListenAndServe(config.SeedServers, false)
	}()
	os.MkdirAll(config.DataDir, 0744)
	log.Println("Opening database at ", config.DataDir)
	db, err := datastore.NewLevelDbDatastore(config.DataDir)
	if err != nil {
		panic(err)
	}
	coord := coordinator.NewCoordinatorImpl(db, raftServer, clusterConfig)
	eng, err := engine.NewQueryEngine(coord)
	if err != nil {
		panic(err)
	}
	log.Println()
	adminServer := admin.NewHttpServer(config.AdminAssetsDir, config.AdminHttpPortString())
	log.Println("Starting admin interface on port", config.AdminHttpPort)
	go func() {
		adminServer.ListenAndServe()
	}()
	log.Println("Starting Http Api server on port", config.ApiHttpPort)
	server := http.NewHttpServer(config.ApiHttpPortString(), eng, coord, coord)
	server.ListenAndServe()
}
<commit_msg>add profiling arguments to the server binary.<commit_after>package main

import (
	"admin"
	"api/http"
	"configuration"
	"coordinator"
	"datastore"
	"engine"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strconv"
	"syscall"
)

const (
	version = "dev"
	gitSha  = ""
)

func waitForSignals(shouldStop chan<- bool, stopped <-chan bool) {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGUSR2)
	for {
		sig := <-ch
		fmt.Printf("Received signal: %s", sig.String())
		switch sig {
		case syscall.SIGINT, syscall.SIGTERM:
			shouldStop <- true
			<-stopped
			os.Exit(0)
		}
	}
}

func startProfiler(filename *string) error {
	if filename == nil || *filename == "" {
		return nil
	}

	cpuProfileFile, err := os.Create(*filename)
	if err != nil {
		return err
	}
	runtime.SetCPUProfileRate(500)
	shouldStop := make(chan bool)
	stopped := make(chan bool)

	go waitForSignals(shouldStop, stopped)

	go func() {
		for {
			select {
			case <-shouldStop:
				cpuProfileFile.Close()
				stopped <- true
				break
			default:
				data := runtime.CPUProfile()
				if data == nil {
					break
				}
				cpuProfileFile.Write(data)
			}
		}
	}()
	return nil
}

func main() {
	fileName := flag.String("config", "config.json.sample", "Config file")
	wantsVersion := flag.Bool("v", false, "Get version number")
	pidFile := flag.String("pidfile", "", "the pid file")
	cpuProfiler := flag.String("cpuprofile", "", "filename where cpu profile data will be written")

	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	startProfiler(cpuProfiler)

	if wantsVersion != nil && *wantsVersion {
		fmt.Printf("InfluxDB v%s (git: %s)\n", version, gitSha)
		return
	}
	config := configuration.LoadConfiguration(*fileName)

	if pidFile != nil && *pidFile != "" {
		pid := strconv.Itoa(os.Getpid())
		if err := ioutil.WriteFile(*pidFile, []byte(pid), 0644); err != nil {
			panic(err)
		}
	}

	log.Println("Starting Influx Server...")
	clusterConfig := coordinator.NewClusterConfiguration()
	os.MkdirAll(config.RaftDir, 0744)

	raftServer := coordinator.NewRaftServer(config.RaftDir, "localhost", config.RaftServerPort, clusterConfig)
	go func() {
		raftServer.ListenAndServe(config.SeedServers, false)
	}()
	os.MkdirAll(config.DataDir, 0744)
	log.Println("Opening database at ", config.DataDir)
	db, err := datastore.NewLevelDbDatastore(config.DataDir)
	if err != nil {
		panic(err)
	}
	coord := coordinator.NewCoordinatorImpl(db, raftServer, clusterConfig)
	eng, err := engine.NewQueryEngine(coord)
	if err != nil {
		panic(err)
	}
	log.Println()
	adminServer := admin.NewHttpServer(config.AdminAssetsDir, config.AdminHttpPortString())
	log.Println("Starting admin interface on port", config.AdminHttpPort)
	go func() {
		adminServer.ListenAndServe()
	}()
	log.Println("Starting Http Api server on port", config.ApiHttpPort)
	server := http.NewHttpServer(config.ApiHttpPortString(), eng, coord, coord)
	server.ListenAndServe()
}

{'fraction_non_alphanumeric': 0.1509977370911335, 'fraction_numerical': 0.0061715696358773916, 'mean_word_length': 4.569301260022909, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7420985', 'n_tokens_mistral': 1878, 'n_tokens_neox': 1745, 'n_words': 365}