<commit_before>package log

import(
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"io"
	"bytes"
	"time"
)

const (
	debugLog = iota
	infoLog
	warningLog
	errorLog
	fatalLog
	numSeverity = 5
)

var (
	levels = map[int]string{
		debugLog:    "DEBUG",
		infoLog:     "INFO",
		warningLog:  "WARNING",
		errorLog:    "ERROR",
		fatalLog:	 "FATAL",
	}
)

const (
	namePrefix = "LEVEL"
	callDepth = 2
)

var Log *Logger

func LevelName(level int) string {
	if name, ok := levels[level]; ok {
		return name
	}
	return namePrefix + strconv.Itoa(level)
}

func NameLevel(name string) int {
	for k, v := range levels {
		if v == name {
			return k
		}
	}
	var level int
	if strings.HasPrefix(name, namePrefix) {
		level, _ = strconv.Atoi(name[len(namePrefix):])
	}
	return level
}

func AddBracket(s string) string{
	b := bytes.Buffer{}
    b.WriteString("[")
    b.WriteString(s)
    b.WriteString("]")
    return b.String()
}

type Logger struct {
	level  int
	logger *log.Logger
}

func New(out io.Writer, prefix string, flag, level int) *Logger {
	return &Logger{
		level:  level,
		logger: log.New(out, prefix, flag),
	}
}

func (l *Logger) output(level int, s string) error {
	return l.logger.Output(callDepth, AddBracket(LevelName(level))+" "+s)
}

func (l *Logger) Output(level int, a ...interface{}) error {
	if level >= l.level {
		return l.output(level, fmt.Sprintln(a...))
	}
	return nil
}

func (l *Logger) Debug(a ...interface{}) {
	l.Output(debugLog, a...)
}

func (l *Logger) Info(a ...interface{}) {
	l.Output(infoLog, a...)
}

func (l *Logger) Warning(a ...interface{}) {
	l.Output(warningLog, a...)
}

func (l *Logger) Error(a ...interface{}) {
	l.Output(errorLog, a...)
}

func (l *Logger) Fatal(a ...interface{}) {
	l.Output(fatalLog, a...)
	os.Exit(1)
}

func FileOpen(path string) (*os.File, error) {
	if fi, err := os.Stat(path); err == nil {
		if !fi.IsDir() {
			return nil, fmt.Errorf("open %s: not a directory", path)
		}
	} else if os.IsNotExist(err) {
		if err := os.MkdirAll(path, 0766); err != nil {
			return nil, err
		}
	} else {
		return nil, err
	}

	var currenttime string = time.Now().Format("2006-01-02")

	logfile,err := os.OpenFile(path + currenttime + "_LOG.log", os.O_RDWR| os.O_CREATE, 0666)
	if err != nil{
		fmt.Printf("%s\n", err.Error())
		os.Exit(-1)
	}

	//defer logfile.Close()		

	return logfile, nil
}

func CreatePrintLog(path string){
	logfile, err := FileOpen(path)
	if err != nil {
		fmt.Printf("%s\n", err.Error)
	}
	var printlevel int = 1
	Log = New(logfile, "\r\n", log.Ldate|log.Ltime|log.Llongfile, printlevel)	
}
<commit_msg>Prevent parallel operation on log print<commit_after>package log

import(
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"io"
	"bytes"
	"time"
	"sync"
)

const (
	debugLog = iota
	infoLog
	warningLog
	errorLog
	fatalLog
	numSeverity = 5
)

var (
	levels = map[int]string{
		debugLog:    "DEBUG",
		infoLog:     "INFO",
		warningLog:  "WARNING",
		errorLog:    "ERROR",
		fatalLog:	 "FATAL",
	}
)

const (
	namePrefix = "LEVEL"
	callDepth = 2
)

var Log *Logger
var lock = sync.Mutex{}

func LevelName(level int) string {
	if name, ok := levels[level]; ok {
		return name
	}
	return namePrefix + strconv.Itoa(level)
}

func NameLevel(name string) int {
	for k, v := range levels {
		if v == name {
			return k
		}
	}
	var level int
	if strings.HasPrefix(name, namePrefix) {
		level, _ = strconv.Atoi(name[len(namePrefix):])
	}
	return level
}

func AddBracket(s string) string{
	b := bytes.Buffer{}
    b.WriteString("[")
    b.WriteString(s)
    b.WriteString("]")
    return b.String()
}

type Logger struct {
	level  int
	logger *log.Logger
}

func New(out io.Writer, prefix string, flag, level int) *Logger {
	return &Logger{
		level:  level,
		logger: log.New(out, prefix, flag),
	}
}

func (l *Logger) output(level int, s string) error {
	return l.logger.Output(callDepth, AddBracket(LevelName(level))+" "+s)
}

func (l *Logger) Output(level int, a ...interface{}) error {
	if level >= l.level {
		return l.output(level, fmt.Sprintln(a...))
	}
	return nil
}

func (l *Logger) Debug(a ...interface{}) {
	lock.Lock()
	defer lock.Unlock()
	l.Output(debugLog, a...)
}

func (l *Logger) Info(a ...interface{}) {
	lock.Lock()
	defer lock.Unlock()
	l.Output(infoLog, a...)
}

func (l *Logger) Warning(a ...interface{}) {
	lock.Lock()
	defer lock.Unlock()
	l.Output(warningLog, a...)
}

func (l *Logger) Error(a ...interface{}) {
	lock.Lock()
	defer lock.Unlock()
	l.Output(errorLog, a...)
}

func (l *Logger) Fatal(a ...interface{}) {
	lock.Lock()
	defer lock.Unlock()
	l.Output(fatalLog, a...)
	os.Exit(1)
}

func FileOpen(path string) (*os.File, error) {
	if fi, err := os.Stat(path); err == nil {
		if !fi.IsDir() {
			return nil, fmt.Errorf("open %s: not a directory", path)
		}
	} else if os.IsNotExist(err) {
		if err := os.MkdirAll(path, 0766); err != nil {
			return nil, err
		}
	} else {
		return nil, err
	}

	var currenttime string = time.Now().Format("2006-01-02")

	logfile,err := os.OpenFile(path + currenttime + "_LOG.log", os.O_RDWR| os.O_CREATE, 0666)
	if err != nil{
		fmt.Printf("%s\n", err.Error())
		os.Exit(-1)
	}

	//defer logfile.Close()	

	return logfile, nil
}

func CreatePrintLog(path string){
	logfile, err := FileOpen(path)
	if err != nil {
		fmt.Printf("%s\n", err.Error)
	}
	var printlevel int = 1
	Log = New(logfile, "\r\n", log.Ldate|log.Ltime|log.Llongfile, printlevel)
}

{'fraction_non_alphanumeric': 0.17329862402380067, 'fraction_numerical': 0.007809594644849386, 'mean_word_length': 3.3802931596091206, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1629810', 'n_tokens_mistral': 2225, 'n_tokens_neox': 2132, 'n_words': 538}