<commit_before>package console

import (
	"bufio"
	"fmt"
	"io"
	stdlog "log"
	"os"
	"strconv"

	"github.com/go-playground/log"
)

// FormatFunc is the function that the workers use to create
// a new Formatter per worker allowing reusable go routine safe
// variable to be used within your Formatter function.
type FormatFunc func() Formatter

// Formatter is the function used to format the Redis entry
type Formatter func(e *log.Entry) []byte

const (
	space   = byte(' ')
	equals  = byte('=')
	newLine = byte('\n')
	colon   = byte(':')
	base10  = 10
	v       = "%v"
	gopath  = "GOPATH"
)

// Console is an instance of the console logger
type Console struct {
	buffer          uint
	numWorkers      uint
	colors          [9]log.ANSIEscSeq
	writer          io.Writer
	formatFunc      FormatFunc
	timestampFormat string
	gopath          string
	fileDisplay     log.FilenameDisplay
	displayColor    bool
	redirStdOutput  bool
}

// Colors mapping.
var defaultColors = [...]log.ANSIEscSeq{
	log.DebugLevel:  log.Green,
	log.TraceLevel:  log.White,
	log.InfoLevel:   log.Blue,
	log.NoticeLevel: log.LightCyan,
	log.WarnLevel:   log.Yellow,
	log.ErrorLevel:  log.LightRed,
	log.PanicLevel:  log.Red,
	log.AlertLevel:  log.Red + log.Underscore,
	log.FatalLevel:  log.Red + log.Underscore + log.Blink,
}

// New returns a new instance of the console logger
func New() *Console {
	c := &Console{
		buffer:          0,
		numWorkers:      1,
		colors:          defaultColors,
		writer:          os.Stderr,
		timestampFormat: log.DefaultTimeFormat,
		displayColor:    true,
		fileDisplay:     log.Lshortfile,
	}

	c.formatFunc = c.defaultFormatFunc

	return c
}

// SetFilenameDisplay tells Console the filename, when present, how to display
func (c *Console) SetFilenameDisplay(fd log.FilenameDisplay) {
	c.fileDisplay = fd
}

// RedirectSTDLogOutput tells Console to redirect the std Logger output
// to the log package itself.
func (c *Console) RedirectSTDLogOutput(b bool) {
	c.redirStdOutput = b
}

// DisplayColor tells Console to output in color or not
// Default is : true
func (c *Console) DisplayColor(color bool) {
	c.displayColor = color
}

// SetTimestampFormat sets Console's timestamp output format
// Default is : "2006-01-02T15:04:05.000000000Z07:00"
func (c *Console) SetTimestampFormat(format string) {
	c.timestampFormat = format
}

// SetWriter sets Console's wriiter
// Default is : os.Stderr
func (c *Console) SetWriter(w io.Writer) {
	c.writer = w
}

// SetBuffersAndWorkers sets the channels buffer size and number of concurrent workers.
// These settings should be thought about together, hence setting both in the same function.
func (c *Console) SetBuffersAndWorkers(size uint, workers uint) {
	c.buffer = size

	if workers == 0 {
		// just in case no log registered yet
		stdlog.Println("Invalid number of workers specified, setting to 1")
		log.Warn("Invalid number of workers specified, setting to 1")

		workers = 1
	}

	c.numWorkers = workers
}

// SetFormatFunc sets FormatFunc each worker will call to get
// a Formatter func
func (c *Console) SetFormatFunc(fn FormatFunc) {
	c.formatFunc = fn
}

// Run starts the logger consuming on the returned channed
func (c *Console) Run() chan<- *log.Entry {

	// pre-setup
	if c.fileDisplay == log.Llongfile {
		// gather $GOPATH for use in stripping off of full name
		// if not found still ok as will be blank
		c.gopath = os.Getenv(gopath)
		if len(c.gopath) != 0 {
			c.gopath += string(os.PathSeparator) + "src" + string(os.PathSeparator)
		}
	}

	// in a big high traffic app, set a higher buffer
	ch := make(chan *log.Entry, c.buffer)

	for i := 0; i <= int(c.numWorkers); i++ {
		go c.handleLog(ch)
	}

	// let's check to see if we should hanlde the std log messages as well
	if c.redirStdOutput {

		done := make(chan struct{})

		go handleStdLogger(done)

		<-done // have to wait, it was running too quickly and some messages can be lost
	}

	return ch
}

// this will redirect the output of
func handleStdLogger(done chan<- struct{}) {
	r, w := io.Pipe()
	defer r.Close()
	defer w.Close()

	stdlog.SetOutput(w)

	scanner := bufio.NewScanner(r)

	go func() {
		done <- struct{}{}
	}()

	for scanner.Scan() {
		log.WithFields(log.F("stdlog", true)).Info(scanner.Text())
	}
}

// handleLog consumes and logs any Entry's passed to the channel
func (c *Console) handleLog(entries <-chan *log.Entry) {

	var e *log.Entry
	// var b io.WriterTo
	var b []byte
	formatter := c.formatFunc()

	for e = range entries {

		b = formatter(e)
		c.writer.Write(b)
		// b.WriteTo(c.writer)

		e.Consumed()
	}
}

func (c *Console) defaultFormatFunc() Formatter {

	var b []byte
	var file string
	var lvl string
	var i int

	if c.displayColor {<commit_msg>updated std log messages to be NOTICE instead of INFO<commit_after>package console

import (
	"bufio"
	"fmt"
	"io"
	stdlog "log"
	"os"
	"strconv"

	"github.com/go-playground/log"
)

// FormatFunc is the function that the workers use to create
// a new Formatter per worker allowing reusable go routine safe
// variable to be used within your Formatter function.
type FormatFunc func() Formatter

// Formatter is the function used to format the Redis entry
type Formatter func(e *log.Entry) []byte

const (
	space   = byte(' ')
	equals  = byte('=')
	newLine = byte('\n')
	colon   = byte(':')
	base10  = 10
	v       = "%v"
	gopath  = "GOPATH"
)

// Console is an instance of the console logger
type Console struct {
	buffer          uint
	numWorkers      uint
	colors          [9]log.ANSIEscSeq
	writer          io.Writer
	formatFunc      FormatFunc
	timestampFormat string
	gopath          string
	fileDisplay     log.FilenameDisplay
	displayColor    bool
	redirStdOutput  bool
}

// Colors mapping.
var defaultColors = [...]log.ANSIEscSeq{
	log.DebugLevel:  log.Green,
	log.TraceLevel:  log.White,
	log.InfoLevel:   log.Blue,
	log.NoticeLevel: log.LightCyan,
	log.WarnLevel:   log.Yellow,
	log.ErrorLevel:  log.LightRed,
	log.PanicLevel:  log.Red,
	log.AlertLevel:  log.Red + log.Underscore,
	log.FatalLevel:  log.Red + log.Underscore + log.Blink,
}

// New returns a new instance of the console logger
func New() *Console {
	c := &Console{
		buffer:          0,
		numWorkers:      1,
		colors:          defaultColors,
		writer:          os.Stderr,
		timestampFormat: log.DefaultTimeFormat,
		displayColor:    true,
		fileDisplay:     log.Lshortfile,
	}

	c.formatFunc = c.defaultFormatFunc

	return c
}

// SetFilenameDisplay tells Console the filename, when present, how to display
func (c *Console) SetFilenameDisplay(fd log.FilenameDisplay) {
	c.fileDisplay = fd
}

// RedirectSTDLogOutput tells Console to redirect the std Logger output
// to the log package itself.
func (c *Console) RedirectSTDLogOutput(b bool) {
	c.redirStdOutput = b
}

// DisplayColor tells Console to output in color or not
// Default is : true
func (c *Console) DisplayColor(color bool) {
	c.displayColor = color
}

// SetTimestampFormat sets Console's timestamp output format
// Default is : "2006-01-02T15:04:05.000000000Z07:00"
func (c *Console) SetTimestampFormat(format string) {
	c.timestampFormat = format
}

// SetWriter sets Console's wriiter
// Default is : os.Stderr
func (c *Console) SetWriter(w io.Writer) {
	c.writer = w
}

// SetBuffersAndWorkers sets the channels buffer size and number of concurrent workers.
// These settings should be thought about together, hence setting both in the same function.
func (c *Console) SetBuffersAndWorkers(size uint, workers uint) {
	c.buffer = size

	if workers == 0 {
		// just in case no log registered yet
		stdlog.Println("Invalid number of workers specified, setting to 1")
		log.Warn("Invalid number of workers specified, setting to 1")

		workers = 1
	}

	c.numWorkers = workers
}

// SetFormatFunc sets FormatFunc each worker will call to get
// a Formatter func
func (c *Console) SetFormatFunc(fn FormatFunc) {
	c.formatFunc = fn
}

// Run starts the logger consuming on the returned channed
func (c *Console) Run() chan<- *log.Entry {

	// pre-setup
	if c.fileDisplay == log.Llongfile {
		// gather $GOPATH for use in stripping off of full name
		// if not found still ok as will be blank
		c.gopath = os.Getenv(gopath)
		if len(c.gopath) != 0 {
			c.gopath += string(os.PathSeparator) + "src" + string(os.PathSeparator)
		}
	}

	// in a big high traffic app, set a higher buffer
	ch := make(chan *log.Entry, c.buffer)

	for i := 0; i <= int(c.numWorkers); i++ {
		go c.handleLog(ch)
	}

	// let's check to see if we should hanlde the std log messages as well
	if c.redirStdOutput {

		done := make(chan struct{})

		go handleStdLogger(done)

		<-done // have to wait, it was running too quickly and some messages can be lost
	}

	return ch
}

// this will redirect the output of
func handleStdLogger(done chan<- struct{}) {
	r, w := io.Pipe()
	defer r.Close()
	defer w.Close()

	stdlog.SetOutput(w)

	scanner := bufio.NewScanner(r)

	go func() {
		done <- struct{}{}
	}()

	for scanner.Scan() {
		log.WithFields(log.F("stdlog", true)).Notice(scanner.Text())
	}
}

// handleLog consumes and logs any Entry's passed to the channel
func (c *Console) handleLog(entries <-chan *log.Entry) {

	var e *log.Entry
	// var b io.WriterTo
	var b []byte
	formatter := c.formatFunc()

	for e = range entries {

		b = formatter(e)
		c.writer.Write(b)
		// b.WriteTo(c.writer)

		e.Consumed()
	}
}

func (c *Console) defaultFormatFunc() Formatter {

	var b []byte
	var file string
	var lvl string
	var i int

	if c.displayColor {
{'fraction_non_alphanumeric': 0.10897571277719113, 'fraction_numerical': 0.008447729672650475, 'mean_word_length': 3.2894021739130435, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1047016', 'n_tokens_mistral': 3325, 'n_tokens_neox': 3139, 'n_words': 1053}