<commit_before>package log

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"time"
)

type Level int32

const (
	FATAL Level = iota - 4
	ERROR
	WARNING
	INFO
	DEBUG
	VDEBUG
	VVDEBUG
	VVVDEBUG
)

var levelNames = map[Level]string{
	FATAL:    "fatal",
	ERROR:    "error",
	WARNING:  "warning",
	INFO:     "info",
	DEBUG:    "debug0",
	VDEBUG:   "debug1",
	VVDEBUG:  "debug2",
	VVVDEBUG: "debug3",
}

type Logger struct {
	lvl Level
	w   io.Writer
}

func New(lvl Level) *Logger {
	return &Logger{lvl, os.Stderr}
}

func (l *Logger) print(lvl Level, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintln(buf, args...)
	l.output(lvl, buf.Bytes())
}

func (l *Logger) printf(lvl Level, format string, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintf(buf, format, args...)

	if buf.Len() == 0 {
		return
	}

	if buf.Bytes()[buf.Len()-1] != '\n' {
		buf.WriteByte('\n')
	}
	l.output(lvl, buf.Bytes())
}

func (l *Logger) output(lvl Level, buf []byte) {
	if lvl > l.lvl {
		return
	}

	h := l.header(lvl)
	l.w.Write(h)
	l.w.Write(buf)
}

func (l *Logger) header(lvl Level) []byte {
	// header format:
	//  YY/MM/DD HH:MM:SS.UUUUUU file:line] LEVEL:

	t := time.Now()
	year, month, day := t.Date()
	year %= 1000
	hour, minute, second := t.Clock()
	usecond := t.Nanosecond() / 1e3

	_, file, line, ok := runtime.Caller(4)
	if !ok {
		file, line = "???", 0
	} else {
		index := strings.LastIndex(file, "/")
		if index != -1 && index != len(file) {
			file = file[index+1:]
		}
	}<commit_msg>atomic writes to writer<commit_after>package log

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

type Level int32

const (
	FATAL Level = iota - 4
	ERROR
	WARNING
	INFO
	DEBUG
	VDEBUG
	VVDEBUG
	VVVDEBUG
)

var levelNames = map[Level]string{
	FATAL:    "fatal",
	ERROR:    "error",
	WARNING:  "warning",
	INFO:     "info",
	DEBUG:    "debug0",
	VDEBUG:   "debug1",
	VVDEBUG:  "debug2",
	VVVDEBUG: "debug3",
}

type Logger struct {
	mu  sync.Mutex
	lvl Level
	w   io.Writer
}

func New(lvl Level) *Logger {
	return &Logger{lvl: lvl, w: os.Stderr}
}

func (l *Logger) print(lvl Level, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintln(buf, args...)
	l.output(lvl, buf.Bytes())
}

func (l *Logger) printf(lvl Level, format string, args ...interface{}) {
	buf := &bytes.Buffer{}
	fmt.Fprintf(buf, format, args...)

	if buf.Len() == 0 {
		return
	}

	if buf.Bytes()[buf.Len()-1] != '\n' {
		buf.WriteByte('\n')
	}
	l.output(lvl, buf.Bytes())
}

func (l *Logger) output(lvl Level, buf []byte) {
	if lvl > l.lvl {
		return
	}

	h := l.header(lvl)

	l.mu.Lock()
	defer l.mu.Unlock()

	l.w.Write(h)
	l.w.Write(buf)
}

func (l *Logger) header(lvl Level) []byte {
	// header format:
	//  YY/MM/DD HH:MM:SS.UUUUUU file:line] LEVEL:

	t := time.Now()
	year, month, day := t.Date()
	year %= 1000
	hour, minute, second := t.Clock()
	usecond := t.Nanosecond() / 1e3

	_, file, line, ok := runtime.Caller(4)
	if !ok {
		file, line = "???", 0
	} else {
		index := strings.LastIndex(file, "/")
		if index != -1 && index != len(file) {
			file = file[index+1:]
		}
	}
{'fraction_non_alphanumeric': 0.1920593931568754, 'fraction_numerical': 0.012265978050355068, 'mean_word_length': 3.0992063492063493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1471375', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1329, 'n_words': 323}