<commit_before>package main

import (
	"io"
	"os"
)

// These are the environmental variables that determine if we log, and if
// we log whether or not the log should go to a file.
const EnvLog = "TF_LOG"          //Set to True
const EnvLogFile = "TF_LOG_PATH" //Set to a file

// logOutput determines where we should send logs (if anywhere).
func logOutput() (logOutput io.Writer, err error) {
	logOutput = nil
	if os.Getenv(EnvLog) != "" {
		logOutput = os.Stderr

		if logPath := os.Getenv(EnvLogFile); logPath != "" {
			var err error
			logOutput, err = os.Create(logPath)
			if err != nil {
				return nil, err
			}
		}
	}

	return
}<commit_msg>log: support for user defined log levels<commit_after>package main

import (
	"io"
	"log"
	"os"
	"strings"

	"github.com/hashicorp/logutils"
)

// These are the environmental variables that determine if we log, and if
// we log whether or not the log should go to a file.
const (
	EnvLog     = "TF_LOG"      // Set to True
	EnvLogFile = "TF_LOG_PATH" // Set to a file
)

var validLevels = []logutils.LogLevel{"TRACE", "DEBUG", "INFO", "WARN", "ERROR"}

// logOutput determines where we should send logs (if anywhere) and the log level.
func logOutput() (logOutput io.Writer, err error) {
	logOutput = nil
	envLevel := os.Getenv(EnvLog)
	if envLevel == "" {
		return
	}

	logOutput = os.Stderr
	if logPath := os.Getenv(EnvLogFile); logPath != "" {
		var err error
		logOutput, err = os.Create(logPath)
		if err != nil {
			return nil, err
		}
	}

	// This was the default since the beginning
	logLevel := logutils.LogLevel("TRACE")

	if isValidLogLevel(envLevel) {
		// allow following for better ux: info, Info or INFO
		logLevel = logutils.LogLevel(strings.ToUpper(envLevel))
	} else {
		log.Printf("[WARN] Invalid log level: %q. Defaulting to level: TRACE. Valid levels are: %+v",
			envLevel, validLevels)
	}

	logOutput = &logutils.LevelFilter{
		Levels:   validLevels,
		MinLevel: logLevel,
		Writer:   logOutput,
	}

	return
}

func isValidLogLevel(level string) bool {
	for _, l := range validLevels {
		if strings.ToUpper(level) == string(l) {
			return true
		}
	}

	return false
}
{'fraction_non_alphanumeric': 0.11676786548341897, 'fraction_numerical': 0.0, 'mean_word_length': 3.3536585365853657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5901334', 'n_tokens_mistral': 774, 'n_tokens_neox': 727, 'n_words': 243}