<commit_before>package main

import (
	"bufio"
	"config"
	"flag"
	"fmt"
	logging "github.com/op/go-logging"
	"mstree"
	"os"
	"os/signal"
	"runtime"
	"runtime/debug"
	"syscall"
	"web"
)

const (
	DEFAULT_CONFIG_FILE = "/etc/metricsearch.conf"
)

var (
	logfileName string
	logfile     *os.File
	log         *logging.Logger = logging.MustGetLogger("metricsearch")
)

func reopenLog() {
	var err error
	if logfile != nil {
		logfile.Close()
	}
	logfile, err = os.OpenFile(logfileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
	if err != nil {
		return
	}
	backend := logging.NewLogBackend(logfile, "", 0)
	logging.SetBackend(backend)
}

func hupCatcher() {
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGHUP)
	for _ = range c {
		log.Debug("HUP signal catched, reopening logfile %s", logfileName)
		reopenLog()
	}
}

func main() {
	var format string
	var confFile, reindexFile string
	var stdinImport bool
	flag.StringVar(&confFile, "c", DEFAULT_CONFIG_FILE, "metricsearch config filename")
	flag.StringVar(&reindexFile, "reindex", "", "reindex from plain text metrics file")
	flag.BoolVar(&stdinImport, "stdin", false, "reindex from stdin")
	flag.Parse()

	conf := config.Load(confFile)
	logging.SetLevel(conf.LogLevel, "metricsearch")

	switch conf.Log {
	case "syslog":
		backend, err := logging.NewSyslogBackend("metricsearch")
		if err != nil {
			log.Error(err.Error())
			return
		}
		logging.SetBackend(backend)
		format = "%{program} %{level} %{message}"
	case "":
		format = "%{color}%{level} %{color:reset}%{message}"
	default:
		logfileName = conf.Log
		reopenLog()
		go hupCatcher()
		format = "[%{time:2006-01-02 15:04:05}] %{level} %{message}"
	}
	logging.SetFormatter(logging.MustStringFormatter(format))

	tree, err := mstree.NewTree(conf.IndexDirectory, conf.SyncBufferSize)
	if err != nil {
		log.Critical("No way to continue, exiting.")
		return
	}

	log.Debug("Configuring runtime: GCPercent(%d), MaxCores(%d), MaxThreads(%d)", conf.GCPercent, conf.MaxCores, conf.MaxThreads)
	runtime.GOMAXPROCS(conf.MaxCores)
	debug.SetGCPercent(conf.GCPercent)
	debug.SetMaxThreads(conf.MaxThreads)

	if stdinImport {
		err := tree.DropIndex()
		if err != nil {
			log.Critical("Error dropping index: %s", err.Error())
			return
		}
		sc := bufio.NewScanner(os.Stdin)
		for sc.Scan() {
			tree.Add(sc.Text())
		}
		log.Notice("Reindexing complete")
		return<commit_msg>logging level fixed?<commit_after>package main

import (
	"bufio"
	"config"
	"flag"
	"fmt"
	logging "github.com/op/go-logging"
	"mstree"
	"os"
	"os/signal"
	"runtime"
	"runtime/debug"
	"syscall"
	"web"
)

const (
	DEFAULT_CONFIG_FILE = "/etc/metricsearch.conf"
)

var (
	logfileName string
	logfile     *os.File
	log         *logging.Logger = logging.MustGetLogger("metricsearch")
)

func reopenLog() {
	var err error
	if logfile != nil {
		logfile.Close()
	}
	logfile, err = os.OpenFile(logfileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
	if err != nil {
		return
	}
	backend := logging.NewLogBackend(logfile, "", 0)
	logging.SetBackend(backend)
}

func hupCatcher() {
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGHUP)
	for _ = range c {
		log.Debug("HUP signal catched, reopening logfile %s", logfileName)
		reopenLog()
	}
}

func main() {
	var format string
	var confFile, reindexFile string
	var stdinImport bool
	flag.StringVar(&confFile, "c", DEFAULT_CONFIG_FILE, "metricsearch config filename")
	flag.StringVar(&reindexFile, "reindex", "", "reindex from plain text metrics file")
	flag.BoolVar(&stdinImport, "stdin", false, "reindex from stdin")
	flag.Parse()

	conf := config.Load(confFile)

	switch conf.Log {
	case "syslog":
		backend, err := logging.NewSyslogBackend("metricsearch")
		if err != nil {
			log.Error(err.Error())
			return
		}
		logging.SetBackend(backend)
		format = "%{program} %{level} %{message}"
	case "":
		format = "%{color}%{level} %{color:reset}%{message}"
	default:
		logfileName = conf.Log
		reopenLog()
		go hupCatcher()
		format = "[%{time:2006-01-02 15:04:05}] %{level} %{message}"
	}
	logging.SetFormatter(logging.MustStringFormatter(format))
	logging.SetLevel(conf.LogLevel, "metricsearch")

	tree, err := mstree.NewTree(conf.IndexDirectory, conf.SyncBufferSize)
	if err != nil {
		log.Critical("No way to continue, exiting.")
		return
	}

	log.Debug("Configuring runtime: GCPercent(%d), MaxCores(%d), MaxThreads(%d)", conf.GCPercent, conf.MaxCores, conf.MaxThreads)
	runtime.GOMAXPROCS(conf.MaxCores)
	debug.SetGCPercent(conf.GCPercent)
	debug.SetMaxThreads(conf.MaxThreads)

	if stdinImport {
		err := tree.DropIndex()
		if err != nil {
			log.Critical("Error dropping index: %s", err.Error())
			return
		}
		sc := bufio.NewScanner(os.Stdin)
		for sc.Scan() {
			tree.Add(sc.Text())
		}
		log.Notice("Reindexing complete")
		return
{'fraction_non_alphanumeric': 0.15267652572380755, 'fraction_numerical': 0.007915017704644865, 'mean_word_length': 4.50057273768614, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2535193', 'n_tokens_mistral': 1922, 'n_tokens_neox': 1795, 'n_words': 365}