<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package logging

import (
	"fmt"
	"strings"
)

type level struct {
	scale byte;
	display string;
}

var (
	DEBUG level = level{scale: 7, display: "DEBUG"};
	INFO level = level{scale: 31, display: "INFO"};
	WARN level = level{scale: 63, display: "WARN"};
	ERROR level = level{scale: 127, display: "ERROR"};
	CRITICAL level = level{scale: 255, display: "CRITICAL"};
	loggingLevel = INFO;
)

func Info(msg string, data ...interface{}) {
	log(INFO, msg, data);
}

func Warn(msg string, data ...interface{}) {
	log(WARN, msg, data);
}

func Debug(msg string, data ...interface{}) {
	log(DEBUG, msg, data);
}

func Error(msg string, data ...interface{}) {
	log(ERROR, msg, data);
}

func Critical(msg string, data ...interface{}) {
	log(CRITICAL, msg, data);
}

func Infof(msg string, data ...interface{}) {
	logf(INFO, msg, data);
}

func Warnf(msg string, data ...interface{}) {
	logf(WARN, msg, data);
}

func Debugf(msg string, data ...interface{}) {
	logf(DEBUG, msg, data);
}

func Errorf(msg string, data ...interface{}) {
	logf(ERROR, msg, data);
}

func Criticalf(msg string, data ...interface{}) {
	logf(CRITICAL, msg, data);
}

func SetLevel(lvl level) {
	loggingLevel = lvl;
}

func SetLevelByString(lvl string) {
	switch (strings.ToUpper(lvl)) {
	case "DEBUG":
		SetLevel(DEBUG);
	case "INFO":
		SetLevel(INFO);
	case "WARN":
		SetLevel(WARN);
	case "ERROR":
		SetLevel(ERROR);
	case "CRITICAL":
		SetLevel(CRITICAL);
	}
}

func log(lvl level, msg string, data ...interface{}) {
	if(lvl.scale < loggingLevel.scale) {
		return;
	}
	var dataLength = len(data[0].([]interface{}));
	if(data == nil || dataLength == 0) {
		fmt.Printf("[%s] %s\n", lvl.display, msg);
	} else {
		cast := make([]interface{}, 3);
		cast[0] = lvl.display;
		cast[1] = msg;
		if(dataLength == 1) {
			cast[2] = data[0].([]interface{})[0];
		} else {
			cast[2] = data[0].([]interface{});
		}
		fmt.Printf("[%s] %s\n%v\n", cast...);
	}
}

func logf(lvl level, msg string, data ...interface{}) {
	if(lvl.scale < loggingLevel.scale) {
		return;
	}
	if(data == nil || len(data[0].([]interface{})) == 0) {
		fmt.Printf("[%s] %s\n", lvl.display, msg);
	} else {
		fmt.Printf("[%s] %s\n", lvl.display, fmt.Sprintf(msg, data[0].([]interface{})...));
	}
}<commit_msg>Add file logging and timestamps<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package logging

import (
	"fmt"
	"strings"
	"os"
	"time"
	"path"
)

type level struct {
	scale   byte;
	display string;
}

var (
	DEBUG level = level{scale: 7, display: "DEBUG"};
	INFO level = level{scale: 31, display: "INFO"};
	WARN level = level{scale: 63, display: "WARN"};
	ERROR level = level{scale: 127, display: "ERROR"};
	CRITICAL level = level{scale: 255, display: "CRITICAL"};
	loggingLevel = INFO;
	logFile *os.File
)

const (
	logFileFolder = "logs"
	logFileName = "2006-01-02T15-04-05.log"
	formatNoData = "[%s] [%s] %s\n"
	formatWithData = "[%s] [%s] %s\n%v\n"
)

func init() {
	var err error;
	os.Mkdir(logFileFolder, os.ModeDir);
	logFile, err = os.OpenFile(path.Join(logFileFolder, time.Now().Format(logFileName)), os.O_WRONLY | os.O_CREATE, os.ModeType)
	if (err != nil) {
		Critical("Could not create log file", err);
	}
}

func Info(msg string, data ...interface{}) {
	log(INFO, msg, data);
}

func Warn(msg string, data ...interface{}) {
	log(WARN, msg, data);
}

func Debug(msg string, data ...interface{}) {
	log(DEBUG, msg, data);
}

func Error(msg string, data ...interface{}) {
	log(ERROR, msg, data);
}

func Critical(msg string, data ...interface{}) {
	log(CRITICAL, msg, data);
}

func Infof(msg string, data ...interface{}) {
	logf(INFO, msg, data);
}

func Warnf(msg string, data ...interface{}) {
	logf(WARN, msg, data);
}

func Debugf(msg string, data ...interface{}) {
	logf(DEBUG, msg, data);
}

func Errorf(msg string, data ...interface{}) {
	logf(ERROR, msg, data);
}

func Criticalf(msg string, data ...interface{}) {
	logf(CRITICAL, msg, data);
}

func SetLevel(lvl level) {
	loggingLevel = lvl;
}

func SetLevelByString(lvl string) {
	switch (strings.ToUpper(lvl)) {
	case "DEBUG":
		SetLevel(DEBUG);
	case "INFO":
		SetLevel(INFO);
	case "WARN":
		SetLevel(WARN);
	case "ERROR":
		SetLevel(ERROR);
	case "CRITICAL":
		SetLevel(CRITICAL);
	}
}

func log(lvl level, msg string, data ...interface{}) {
	if (lvl.scale < loggingLevel.scale) {
		return;
	}
	var dataLength = len(data[0].([]interface{}));
	if (data == nil || dataLength == 0) {
		var output = fmt.Sprintf(formatNoData, getTimestamp(), lvl.display, msg);
		fmt.Print(output);
		logFile.WriteString(output);
	} else {
		cast := make([]interface{}, 4);
		cast[0] = getTimestamp();
		cast[1] = lvl.display;
		cast[2] = msg;
		if (dataLength == 1) {
			cast[3] = data[0].([]interface{})[0];
		} else {
			cast[3] = data[0].([]interface{});
		}
		var output = fmt.Sprintf(formatWithData, cast...);
		fmt.Print(output);
		logFile.WriteString(output);
	}
}

func logf(lvl level, msg string, data ...interface{}) {
	if (lvl.scale < loggingLevel.scale) {
		return;
	}
	if (data == nil || len(data[0].([]interface{})) == 0) {
		var output = fmt.Sprintf(formatNoData, getTimestamp(), lvl.display, msg);
		fmt.Print(output);
		logFile.WriteString(output);
	} else {
		var output = fmt.Sprintf(formatNoData, getTimestamp(), lvl.display, fmt.Sprintf(msg, data[0].([]interface{})...));
		fmt.Print(output);
		logFile.WriteString(output);
	}
}

func getTimestamp() string {
	return time.Now().Format("15:04:05");
}
{'fraction_non_alphanumeric': 0.17348032460572654, 'fraction_numerical': 0.0133210840606339, 'mean_word_length': 4.115113547376664, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7613584', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2296, 'n_words': 664}