<commit_before>package lxlog

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"runtime"
	"strings"
	"io"
	"net/http"
	"os/exec"
	"bufio"
)

var GlobalLogLevel Level = InfoLevel

const (
	default_logger = "default_logger"

	PanicLevel = Level("PanicLevel")
	FatalLevel = Level("FatalLevel")
	ErrorLevel = Level("ErrorLevel")
	WarnLevel = Level("WarnLevel")
	InfoLevel = Level("InfoLevel")
	DebugLevel = Level("DebugLevel")
)

var logLevels = map[Level]logrus.Level{
	PanicLevel: logrus.PanicLevel,
	FatalLevel: logrus.FatalLevel,
	ErrorLevel: logrus.ErrorLevel,
	WarnLevel: logrus.WarnLevel,
	InfoLevel: logrus.InfoLevel,
	DebugLevel: logrus.DebugLevel,
}

type Level string

func (level Level) String() string {
	return string(level)
}

type Fields logrus.Fields

type LxLogger struct {
	loggers map[string]*logrus.Logger
	fields  Fields
	err     error
	name	string
}

func New(name string) *LxLogger {
	loggers := make(map[string]*logrus.Logger)
	loggers[default_logger] = logrus.New()
	lxlogger := &LxLogger{
		loggers: loggers,
		name: name,
	}
	lxlogger.SetLogLevel(GlobalLogLevel)
	return lxlogger
}

func (lxlog *LxLogger) WithFields(fields Fields) *LxLogger {
	return &LxLogger{
		loggers: lxlog.loggers,
		fields: fields,
		err: lxlog.err,
		name: lxlog.name,
	}
}

func (lxlog *LxLogger) WithErr(err error) *LxLogger {
	return &LxLogger{
		loggers: lxlog.loggers,
		fields: lxlog.fields,
		err: err,
		name: lxlog.name,
	}
}

func (lxlog *LxLogger) SetLogLevel(level Level) {
	for _, logrusLogger := range lxlog.loggers {
		logrusLogger.Level = logLevels[level]
	}
}

func (lxlog *LxLogger) AddWriter(name string, level Level, w io.Writer) {
	newLogger := logrus.New()
	newLogger.Out = w
	newLogger.Level = logLevels[level]
	lxlog.loggers[name] = newLogger
}

func (lxlog *LxLogger) DeleteWriter(name string) {
	delete(lxlog.loggers, name)
}

func (lxlog *LxLogger) LogCommand(cmd *exec.Cmd, asDebug bool) {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return
	}
	go func() {
		// read command's stdout line by line
		in := bufio.NewScanner(stdout)

		for in.Scan() {
			if asDebug {
				lxlog.Debugf(in.Text())
			} else {
				lxlog.Infof(in.Text())
			}
		}
	}()
	go func() {
		// read command's stdout line by line
		in := bufio.NewScanner(stderr)

		for in.Scan() {
			lxlog.Errorf(in.Text())
		}
	}()
}

func (lxlog *LxLogger) Infof(format string, a ...interface{}) {
	lxlog.log(InfoLevel, format, a...)
}

func (lxlog *LxLogger) Debugf(format string, a ...interface{}) {
	lxlog.log(DebugLevel, format, a...)
}

func (lxlog *LxLogger) Warnf(format string, a ...interface{}) {
	lxlog.log(WarnLevel, format, a...)
}

func (lxlog *LxLogger) Errorf(format string, a ...interface{}) {
	lxlog.log(ErrorLevel, format, a...)
}

func (lxlog *LxLogger) Fatalf(format string, a ...interface{}) {
	lxlog.log(FatalLevel, format, a...)
}

func (lxlog *LxLogger) Panicf(format string, a ...interface{}) {
	lxlog.log(PanicLevel, format, a...)
}

func (lxlog *LxLogger) log(level Level, format string, a ...interface{}) {
	format = lxlog.addTrace(format)
	for _, optionalLog := range lxlog.loggers {
		entry := optionalLog.WithFields(logrus.Fields(lxlog.fields))
		if lxlog.err != nil {
			entry = entry.WithError(lxlog.err)
		}
		switch level {
		case PanicLevel:
			entry.Panicf(format, a...)
			break
		case FatalLevel:
			entry.Fatalf(format, a...)
			break
		case ErrorLevel:
			entry.Errorf(format, a...)
			break
		case WarnLevel:
			entry.Warnf(format, a...)
			break
		case InfoLevel:
			entry.Infof(format, a...)
			break
		case DebugLevel:
			entry.Debugf(format, a...)
			break
		}
		if flusher, ok := optionalLog.Out.(http.Flusher); ok {
			flusher.Flush()
		}
	}
}

func (lxlog *LxLogger) addTrace(format string) string {
	pc, fn, line, _ := runtime.Caller(3)
	pathComponents := strings.Split(fn, "/")
	var truncatedPath string
	if len(pathComponents) > 3 {
		truncatedPath = strings.Join(pathComponents[len(pathComponents) - 2:], "/")
	} else {
		truncatedPath = strings.Join(pathComponents, "/")
	}
	fnName := runtime.FuncForPC(pc).Name()
	fnNameComponents := strings.Split(fnName, "/")
	truncatedFnName := fnNameComponents[len(fnNameComponents) - 1]

	file := fmt.Sprintf("(%s): %s[%s:%d] ", lxlog.name, truncatedFnName, truncatedPath, line)

	return file + format
}<commit_msg>enable adding extra trace levels<commit_after>package lxlog

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"runtime"
	"strings"
	"io"
	"net/http"
	"os/exec"
	"bufio"
)

var GlobalLogLevel Level = InfoLevel

const (
	default_logger = "default_logger"
	default_trace = 3

	PanicLevel = Level("PanicLevel")
	FatalLevel = Level("FatalLevel")
	ErrorLevel = Level("ErrorLevel")
	WarnLevel = Level("WarnLevel")
	InfoLevel = Level("InfoLevel")
	DebugLevel = Level("DebugLevel")
)

var logLevels = map[Level]logrus.Level{
	PanicLevel: logrus.PanicLevel,
	FatalLevel: logrus.FatalLevel,
	ErrorLevel: logrus.ErrorLevel,
	WarnLevel: logrus.WarnLevel,
	InfoLevel: logrus.InfoLevel,
	DebugLevel: logrus.DebugLevel,
}

type Level string

func (level Level) String() string {
	return string(level)
}

type Fields logrus.Fields

type LxLogger struct {
	loggers map[string]*logrus.Logger
	fields  Fields
	err     error
	name	string
	trace   int
}

func New(name string) *LxLogger {
	loggers := make(map[string]*logrus.Logger)
	loggers[default_logger] = logrus.New()
	lxlogger := &LxLogger{
		loggers: loggers,
		name: name,
		trace: 0,
	}
	lxlogger.SetLogLevel(GlobalLogLevel)
	return lxlogger
}

func (lxlog *LxLogger) WithFields(fields Fields) *LxLogger {
	return &LxLogger{
		loggers: lxlog.loggers,
		fields: fields,
		err: lxlog.err,
		name: lxlog.name,
		trace: lxlog.trace,
	}
}

func (lxlog *LxLogger) WithErr(err error) *LxLogger {
	return &LxLogger{
		loggers: lxlog.loggers,
		fields: lxlog.fields,
		err: err,
		name: lxlog.name,
		trace: lxlog.trace,
	}
}

func (lxlog *LxLogger) WithTrace(trace int) *LxLogger {
	return &LxLogger{
		loggers: lxlog.loggers,
		fields: lxlog.fields,
		err: lxlog.err,
		name: lxlog.name,
		trace: trace,
	}
}

func (lxlog *LxLogger) SetLogLevel(level Level) {
	for _, logrusLogger := range lxlog.loggers {
		logrusLogger.Level = logLevels[level]
	}
}

func (lxlog *LxLogger) AddWriter(name string, level Level, w io.Writer) {
	newLogger := logrus.New()
	newLogger.Out = w
	newLogger.Level = logLevels[level]
	lxlog.loggers[name] = newLogger
}

func (lxlog *LxLogger) DeleteWriter(name string) {
	delete(lxlog.loggers, name)
}

func (lxlog *LxLogger) LogCommand(cmd *exec.Cmd, asDebug bool) {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return
	}
	go func() {
		// read command's stdout line by line
		in := bufio.NewScanner(stdout)

		for in.Scan() {
			if asDebug {
				lxlog.Debugf(in.Text())
			} else {
				lxlog.Infof(in.Text())
			}
		}
	}()
	go func() {
		// read command's stdout line by line
		in := bufio.NewScanner(stderr)

		for in.Scan() {
			lxlog.Errorf(in.Text())
		}
	}()
}

func (lxlog *LxLogger) Infof(format string, a ...interface{}) {
	lxlog.log(InfoLevel, format, a...)
}

func (lxlog *LxLogger) Debugf(format string, a ...interface{}) {
	lxlog.log(DebugLevel, format, a...)
}

func (lxlog *LxLogger) Warnf(format string, a ...interface{}) {
	lxlog.log(WarnLevel, format, a...)
}

func (lxlog *LxLogger) Errorf(format string, a ...interface{}) {
	lxlog.log(ErrorLevel, format, a...)
}

func (lxlog *LxLogger) Fatalf(format string, a ...interface{}) {
	lxlog.log(FatalLevel, format, a...)
}

func (lxlog *LxLogger) Panicf(format string, a ...interface{}) {
	lxlog.log(PanicLevel, format, a...)
}

func (lxlog *LxLogger) log(level Level, format string, a ...interface{}) {
	format = lxlog.addTrace(format)
	for _, optionalLog := range lxlog.loggers {
		entry := optionalLog.WithFields(logrus.Fields(lxlog.fields))
		if lxlog.err != nil {
			entry = entry.WithError(lxlog.err)
		}
		switch level {
		case PanicLevel:
			entry.Panicf(format, a...)
			break
		case FatalLevel:
			entry.Fatalf(format, a...)
			break
		case ErrorLevel:
			entry.Errorf(format, a...)
			break
		case WarnLevel:
			entry.Warnf(format, a...)
			break
		case InfoLevel:
			entry.Infof(format, a...)
			break
		case DebugLevel:
			entry.Debugf(format, a...)
			break
		}
		if flusher, ok := optionalLog.Out.(http.Flusher); ok {
			flusher.Flush()
		}
	}
}

func (lxlog *LxLogger) addTrace(format string) string {
	pc, fn, line, _ := runtime.Caller(default_trace+lxlog.trace)
	pathComponents := strings.Split(fn, "/")
	var truncatedPath string
	if len(pathComponents) > 3 {
		truncatedPath = strings.Join(pathComponents[len(pathComponents) - 2:], "/")
	} else {
		truncatedPath = strings.Join(pathComponents, "/")
	}
	fnName := runtime.FuncForPC(pc).Name()
	fnNameComponents := strings.Split(fnName, "/")
	truncatedFnName := fnNameComponents[len(fnNameComponents) - 1]

	file := fmt.Sprintf("(%s): %s[%s:%d] ", lxlog.name, truncatedFnName, truncatedPath, line)

	return file + format
}
{'fraction_non_alphanumeric': 0.14829570606463036, 'fraction_numerical': 0.00099601593625498, 'mean_word_length': 3.990060739922695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6460319', 'n_tokens_mistral': 3681, 'n_tokens_neox': 3419, 'n_words': 762}