<commit_before>package logger

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
)

// StdlibLogger is an implementation of the Logger interface that uses the stdlibs loggger for logging.
type StdlibLogger struct {
	logger *log.Logger
	level  LogLevel
}

// to see if we implement Logger
var _ Logger = NewStdlibLogger()

// NewStdlibLogger returns a new StdlibLogger.
// If the environment variable LOGGER_DISCARD is set, the logger will discard everything (useful for benchmarks)
func NewStdlibLogger() *StdlibLogger {
	if os.Getenv("LOGGER_DISCARD") != "" {
		// Hack to completely disable logging, for example when running benchmarks.
		return &StdlibLogger{
			logger: log.New(ioutil.Discard, "", 0),
			level:  Off,
		}
	}

	return &StdlibLogger{
		logger: log.New(os.Stdout, "", log.Ltime),
		level:  LevelOK,
	}
}

// See Logger.SetLevel
func (l *StdlibLogger) SetLevel(level LogLevel) {
	if level < Off || level > NumLevels {
		panic("Invalid log level")
	}
	l.level = level
}

// See log.SetOutput
func (l *StdlibLogger) SetOutput(w io.Writer) {
	l.logger.SetOutput(w)
}

// See log.SetFlags
func (l *StdlibLogger) SetFlags(flag int) {
	l.logger.SetFlags(flag)
}

// See log.SetPrefix
func (l *StdlibLogger) SetPrefix(prefix string) {
	l.logger.SetPrefix(prefix)
}

// See Logger.Logs
func (l *StdlibLogger) Logs(level LogLevel) bool {
	return level >= l.level
}

// Debugln logs a line with a DEBUG prefix.
func (l *StdlibLogger) Debugln(vals ...interface{}) {
	if !l.Logs(LevelDebug) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "DEBUG: "+s)
}

// Debugf logs a formatted line with a DEBUG prefix.
func (l *StdlibLogger) Debugf(format string, vals ...interface{}) {
	if !l.Logs(LevelDebug) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "DEBUG: "+s)
}

// Infoln logs a line with a VERBOSE prefix.
func (l *StdlibLogger) Verboseln(vals ...interface{}) {
	if !l.Logs(LevelVerbose) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "VERBOSE: "+s)
}

// Infof logs a formatted line with a VERBOSE prefix.
func (l *StdlibLogger) Verbosef(format string, vals ...interface{}) {
	if !l.Logs(LevelVerbose) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "VERBOSE: "+s)
}

// Infoln logs a line with an INFO prefix.
func (l *StdlibLogger) Infoln(vals ...interface{}) {
	if !l.Logs(LevelInfo) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "INFO: "+s)
}

// Infof logs a formatted line with an INFO prefix.
func (l *StdlibLogger) Infof(format string, vals ...interface{}) {
	if !l.Logs(LevelInfo) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "INFO: "+s)
}

// Okln logs a line with an OK prefix.
func (l *StdlibLogger) Okln(vals ...interface{}) {
	if !l.Logs(LevelOK) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "OK: "+s)
}

// Okf logs a formatted line with an OK prefix.
func (l *StdlibLogger) Okf(format string, vals ...interface{}) {
	if !l.Logs(LevelOK) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "OK: "+s)
}

// Warnln logs a line with a WARNING prefix.
func (l *StdlibLogger) Warnln(vals ...interface{}) {
	if !l.Logs(LevelWarn) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "WARNING: "+s)
}

// Warnf logs a formatted line with a WARNING prefix.
func (l *StdlibLogger) Warnf(format string, vals ...interface{}) {
	if !l.Logs(LevelWarn) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "WARNING: "+s)
}

// Fatalln logs a line with a FATAL prefix and exits the process with exit code 1.
func (l *StdlibLogger) Fatalln(vals ...interface{}) {
	if !l.Logs(LevelFatal) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "FATAL: "+s)
	os.Exit(1)
}

// Fatalf logs a formatted line with a FATAL prefix and exits the process with exit code 1.
func (l *StdlibLogger) Fatalf(format string, vals ...interface{}) {
	if !l.Logs(LevelFatal) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "FATAL: "+s)
	os.Exit(1)
}
<commit_msg>godoc<commit_after>package logger

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
)

// StdlibLogger is an implementation of the Logger interface that uses the stdlibs loggger for logging.
type StdlibLogger struct {
	logger *log.Logger
	level  LogLevel
}

// to see if we implement Logger
var _ Logger = NewStdlibLogger()

// NewStdlibLogger returns a new StdlibLogger that logs to Stdout with a time prefix.
// If the environment variable LOGGER_DISCARD is set, the logger will discard everything (useful for benchmarks)
func NewStdlibLogger() *StdlibLogger {
	if os.Getenv("LOGGER_DISCARD") != "" {
		// Hack to completely disable logging, for example when running benchmarks.
		return &StdlibLogger{
			logger: log.New(ioutil.Discard, "", 0),
			level:  Off,
		}
	}

	return &StdlibLogger{
		logger: log.New(os.Stdout, "", log.Ltime),
		level:  LevelOK,
	}
}

// See Logger.SetLevel
func (l *StdlibLogger) SetLevel(level LogLevel) {
	if level < Off || level > NumLevels {
		panic("Invalid log level")
	}
	l.level = level
}

// See log.SetOutput
func (l *StdlibLogger) SetOutput(w io.Writer) {
	l.logger.SetOutput(w)
}

// See log.SetFlags
func (l *StdlibLogger) SetFlags(flag int) {
	l.logger.SetFlags(flag)
}

// See log.SetPrefix
func (l *StdlibLogger) SetPrefix(prefix string) {
	l.logger.SetPrefix(prefix)
}

// See Logger.Logs
func (l *StdlibLogger) Logs(level LogLevel) bool {
	return level >= l.level
}

// Debugln logs a line with a DEBUG prefix.
func (l *StdlibLogger) Debugln(vals ...interface{}) {
	if !l.Logs(LevelDebug) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "DEBUG: "+s)
}

// Debugf logs a formatted line with a DEBUG prefix.
func (l *StdlibLogger) Debugf(format string, vals ...interface{}) {
	if !l.Logs(LevelDebug) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "DEBUG: "+s)
}

// Infoln logs a line with a VERBOSE prefix.
func (l *StdlibLogger) Verboseln(vals ...interface{}) {
	if !l.Logs(LevelVerbose) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "VERBOSE: "+s)
}

// Infof logs a formatted line with a VERBOSE prefix.
func (l *StdlibLogger) Verbosef(format string, vals ...interface{}) {
	if !l.Logs(LevelVerbose) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "VERBOSE: "+s)
}

// Infoln logs a line with an INFO prefix.
func (l *StdlibLogger) Infoln(vals ...interface{}) {
	if !l.Logs(LevelInfo) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "INFO: "+s)
}

// Infof logs a formatted line with an INFO prefix.
func (l *StdlibLogger) Infof(format string, vals ...interface{}) {
	if !l.Logs(LevelInfo) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "INFO: "+s)
}

// Okln logs a line with an OK prefix.
func (l *StdlibLogger) Okln(vals ...interface{}) {
	if !l.Logs(LevelOK) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "OK: "+s)
}

// Okf logs a formatted line with an OK prefix.
func (l *StdlibLogger) Okf(format string, vals ...interface{}) {
	if !l.Logs(LevelOK) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "OK: "+s)
}

// Warnln logs a line with a WARNING prefix.
func (l *StdlibLogger) Warnln(vals ...interface{}) {
	if !l.Logs(LevelWarn) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "WARNING: "+s)
}

// Warnf logs a formatted line with a WARNING prefix.
func (l *StdlibLogger) Warnf(format string, vals ...interface{}) {
	if !l.Logs(LevelWarn) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "WARNING: "+s)
}

// Fatalln logs a line with a FATAL prefix and exits the process with exit code 1.
func (l *StdlibLogger) Fatalln(vals ...interface{}) {
	if !l.Logs(LevelFatal) {
		return
	}
	s := fmt.Sprintln(vals...)
	l.logger.Output(2, "FATAL: "+s)
	os.Exit(1)
}

// Fatalf logs a formatted line with a FATAL prefix and exits the process with exit code 1.
func (l *StdlibLogger) Fatalf(format string, vals ...interface{}) {
	if !l.Logs(LevelFatal) {
		return
	}
	s := fmt.Sprintf(format, vals...)
	l.logger.Output(2, "FATAL: "+s)
	os.Exit(1)
}

{'fraction_non_alphanumeric': 0.157369782046546, 'fraction_numerical': 0.004186676517670238, 'mean_word_length': 3.55524397083567, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1405729', 'n_tokens_mistral': 3156, 'n_tokens_neox': 3071, 'n_words': 876}