<commit_before>package node.util

import jet.modules.Module
import java.util.logging.Logger
import java.util.logging.Level
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.node.ObjectNode
import java.util.Date
import java.text.SimpleDateFormat
import java.util.logging.Handler
import java.util.logging.LogRecord
import java.util.logging.LogManager
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ArrayNode
import java.util.ArrayList
import java.util.Comparator
import java.util.logging.Formatter
import java.io.PrintWriter
import java.io.StringWriter

private var inited = false
fun init() {
  if (!inited) {
    val log = LogManager.getLogManager()!!.getLogger("")!!
    for (handler in log.getHandlers()) {
      handler.setFormatter(LogFormat)
    }<commit_msg>Allow logger to be configured by standard configuration scheme.<commit_after>package node.util

import jet.modules.Module
import java.util.logging.Logger
import java.util.logging.Level
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.node.ObjectNode
import java.util.Date
import java.text.SimpleDateFormat
import java.util.logging.Handler
import java.util.logging.LogRecord
import java.util.logging.LogManager
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ArrayNode
import java.util.ArrayList
import java.util.Comparator
import java.util.logging.Formatter
import java.io.PrintWriter
import java.io.StringWriter
import node.Configuration
import java.util.logging.Filter

private var inited = false
fun init() {
  if (!inited) {
    val logManager = LogManager.getLogManager()!!
    val log = logManager.getLogger("")!!

    val packageLevels = Configuration.get("logging.packages") as? Map<String, String>

    val globalLevel = Level.parse(Configuration.get("logging.global.level") as? String ?: "INFO")
    log.setLevel(globalLevel)
    for (handler in log.getHandlers()) {
      handler.setFormatter(LogFormat)
      handler.setLevel(globalLevel)

      handler.setFilter(object: Filter {
        public override fun isLoggable(record: LogRecord?): Boolean {
          try {
          if (packageLevels != null) {
            for (l in packageLevels) {
              val loggerName = record!!.getLoggerName() ?: ""
              if (loggerName.startsWith(l.key)) {
                val level = Level.parse(l.value)
                if (record.getLevel()!!.intValue() > level.intValue()) {
                  return true
                }
                return false
              }
            }
          }
          return true
          } catch (t: Throwable) {
            t.printStackTrace()
            return false
          }
        }
      })
    }
{'fraction_non_alphanumeric': 0.10078909612625538, 'fraction_numerical': 0.0, 'mean_word_length': 3.156482861400894, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3854125', 'n_tokens_mistral': 807, 'n_tokens_neox': 757, 'n_words': 172}