<commit_before><commit_msg>New: Logger<commit_after>package log

import (
	"bufio"
	"io"
	"fmt"
	"time"
)

func (p *Log) Debug(msg string) {
	p.output(LogLevelDebug, msg)
}

func (p *Log) Info(msg string) {
	p.output(LogLevelInfo, msg)
}

func (p *Log) Warn(msg string) {
	p.output(LogLevelWarn, msg)
}

func (p *Log) Error(msg string) {
	p.output(LogLevelError, msg)
}

func (p *Log) output(level int, msg string) {
	if level < p.level {
		return
	}
	LogLevel(level).String()
	str := fmt.Sprintf("[%s] %s %s", time.Now().Format("2006-01-02 15:04:05"), level, msg)
	p.w.Write([]byte(str))
}

func (p *Log) Close() {
	p.w.Flush()
}

func (p *Log) SetLevel(level int) {
	p.level = level
}

func NewLog(w io.Writer, level int) *Log {
	return &Log{bufio.NewWriter(w), level}
}

type Log struct {
	w *bufio.Writer
	level int
}

func (p LogLevel) String() string {
	switch p {
	case LogLevelDebug:
		return "DEBUG"
	case LogLevelInfo:
		return "INFO"
	case LogLevelWarn:
		return "WARN"
	case LogLevelError:
		return "ERROR"
	}
	return "UNKNOWN"
}

type LogLevel int

const (
	LogLevelDebug = 0
	LogLevelInfo  = 1
	LogLevelWarn  = 2
	LogLevelError = 3
)
{'fraction_non_alphanumeric': 0.15605928509154315, 'fraction_numerical': 0.015693112467306015, 'mean_word_length': 3.991304347826087, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1744648', 'n_tokens_mistral': 493, 'n_tokens_neox': 479, 'n_words': 122}