<commit_before>// Copyright (c) 2014 <NAME>
// Licensed under the MIT license

package main

import (
	"bytes"
	"fmt"
	"github.com/op/go-logging"
	stdlog "log"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	log     = logging.MustGetLogger("main")
	rlogger RuntimeLogger
	dlogger DownloadsLogger
)

type RuntimeLogger struct {
	f *os.File
}

type DownloadsLogger struct {
	sync.RWMutex
	l *stdlog.Logger
	f *os.File
}

// ReloadLogs will reopen the logs to allow rotations
func ReloadLogs() {
	ReloadRuntimeLogs()
	ReloadDownloadLogs()
}

func ReloadRuntimeLogs() {
	logging.SetFormatter(logging.MustStringFormatter("%{time:2006/01/02 15:04:05.000 MST} %{message}"))
	logFlags := 0

	if os.Getenv("DEBUG") != "" {
		logFlags = stdlog.Lshortfile
		logging.SetLevel(logging.DEBUG, "main")
	} else {
		logging.SetLevel(logging.INFO, "main")
	}

	logColor := false

	stat, _ := os.Stdout.Stat()
	if (stat.Mode() & os.ModeCharDevice) != 0 {
		logColor = true //TODO make it optionnal
	}

	if rlogger.f != nil {
		rlogger.f.Close()
	} else {
		rlogger.f = os.Stderr
	}

	if runLog != "" {
		var err error
		rlogger.f, err = os.OpenFile(runLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Cannot open log file for writing")
			rlogger.f = os.Stderr
		} else {
			logColor = false
		}
	}

	logBackend := logging.NewLogBackend(rlogger.f, "", logFlags)
	logBackend.Color = logColor

	logging.SetBackend(logBackend)
}

func ReloadDownloadLogs() {
	dlogger.Lock()
	defer dlogger.Unlock()

	if GetConfig().LogDir == "" {
		if dlogger.f != nil {
			dlogger.f.Close()
		}
		dlogger.f = nil
		dlogger.l = nil
		return
	}

	logfile := GetConfig().LogDir + "/downloads.log"
	createHeader := true

	s, err := os.Stat(logfile)
	if err == nil && s.Size() > 0 {
		createHeader = false
	}

	if dlogger.f != nil {
		dlogger.f.Close()
	}
	dlogger.f, err = os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)<commit_msg>Improve the logger output in debug and non-debug<commit_after>// Copyright (c) 2014 <NAME>
// Licensed under the MIT license

package main

import (
	"bytes"
	"fmt"
	"github.com/op/go-logging"
	stdlog "log"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	log     = logging.MustGetLogger("main")
	rlogger RuntimeLogger
	dlogger DownloadsLogger
)

type RuntimeLogger struct {
	f *os.File
}

type DownloadsLogger struct {
	sync.RWMutex
	l *stdlog.Logger
	f *os.File
}

// ReloadLogs will reopen the logs to allow rotations
func ReloadLogs() {
	ReloadRuntimeLogs()
	ReloadDownloadLogs()
}

func ReloadRuntimeLogs() {
	logColor := false

	stat, _ := os.Stdout.Stat()
	if (stat.Mode() & os.ModeCharDevice) != 0 {
		logColor = true //TODO make it optionnal
	}

	if rlogger.f != nil {
		rlogger.f.Close()
	} else {
		rlogger.f = os.Stderr
	}

	if runLog != "" {
		var err error
		rlogger.f, err = os.OpenFile(runLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Cannot open log file for writing")
			rlogger.f = os.Stderr
		} else {
			logColor = false
		}
	}

	logBackend := logging.NewLogBackend(rlogger.f, "", 0)
	logBackend.Color = logColor

	logging.SetBackend(logBackend)

	if debug {
		logging.SetFormatter(logging.MustStringFormatter("%{shortfile:-20s}%{time:2006/01/02 15:04:05.000 MST} %{message}"))
		logging.SetLevel(logging.DEBUG, "main")
	} else {
		logging.SetFormatter(logging.MustStringFormatter("%{time:2006/01/02 15:04:05.000 MST} %{message}"))
		logging.SetLevel(logging.INFO, "main")
	}
}

func ReloadDownloadLogs() {
	dlogger.Lock()
	defer dlogger.Unlock()

	if GetConfig().LogDir == "" {
		if dlogger.f != nil {
			dlogger.f.Close()
		}
		dlogger.f = nil
		dlogger.l = nil
		return
	}

	logfile := GetConfig().LogDir + "/downloads.log"
	createHeader := true

	s, err := os.Stat(logfile)
	if err == nil && s.Size() > 0 {
		createHeader = false
	}

	if dlogger.f != nil {
		dlogger.f.Close()
	}
	dlogger.f, err = os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
{'fraction_non_alphanumeric': 0.1480836236933798, 'fraction_numerical': 0.020657043305126927, 'mean_word_length': 3.630184331797235, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6446985', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1571, 'n_words': 338}