<commit_before>module Ougai
  module Logging
    attr_accessor :with_fields
    attr_writer :before_log

    def debug(message = nil, ex = nil, data = nil, &block)
      return true if level > Logger::DEBUG
      args = block ? yield : [message, ex, data]
      append(Logger::DEBUG, args)
    end

    def info(message = nil, ex = nil, data = nil, &block)
      return true if level > Logger::INFO
      args = block ? yield : [message, ex, data]
      append(Logger::INFO, args)
    end

    def warn(message = nil, ex = nil, data = nil, &block)
      return true if level > Logger::WARN
      args = block ? yield : [message, ex, data]
      append(Logger::WARN, args)
    end

    def error(message = nil, ex = nil, data = nil, &block)
      return true if level > Logger::ERROR
      args = block ? yield : [message, ex, data]
      append(Logger::ERROR, args)
    end

    def fatal(message = nil, ex = nil, data = nil, &block)
      return true if level > Logger::FATAL
      args = block ? yield : [message, ex, data]
      append(Logger::FATAL, args)
    end

    def unknown(message = nil, ex = nil, data = nil, &block)
      args = block ? yield : [message, ex, data]
      append(Logger::UNKNOWN, args)
    end

    def child(fields = {})
      ChildLogger.new(self, fields)
    end

    def chain(_severity, _args, _fields, _hooks)
      raise NotImplementedError
    end

    protected

    def append(severity, args)
      raise NotImplementedError
    end

    def merge_fields(base_data, new_data)
      base_data.merge(new_data) do |_, base_val, new_val|
        if base_val.is_a?(Array) and new_val.is_a?(Array)
          (base_val + new_val).uniq
        else
          new_val
        end
      end
    end
  end
end<commit_msg>Refactor logging<commit_after>module Ougai
  module Logging
    attr_accessor :with_fields
    attr_writer :before_log

    def debug(message = nil, ex = nil, data = nil, &block)
      log(Logger::DEBUG, message, ex, data, block)
    end

    def info(message = nil, ex = nil, data = nil, &block)
      log(Logger::INFO, message, ex, data, block)
    end

    def warn(message = nil, ex = nil, data = nil, &block)
      log(Logger::WARN, message, ex, data, block)
    end

    def error(message = nil, ex = nil, data = nil, &block)
      log(Logger::ERROR, message, ex, data, block)
    end

    def fatal(message = nil, ex = nil, data = nil, &block)
      log(Logger::FATAL, message, ex, data, block)
    end

    def unknown(message = nil, ex = nil, data = nil, &block)
      args = block ? yield : [message, ex, data]
      append(Logger::UNKNOWN, args)
    end

    def child(fields = {})
      ChildLogger.new(self, fields)
    end

    def chain(_severity, _args, _fields, _hooks)
      raise NotImplementedError
    end

    protected

    def append(severity, args)
      raise NotImplementedError
    end

    def merge_fields(base_data, new_data)
      base_data.merge(new_data) do |_, base_val, new_val|
        if base_val.is_a?(Array) and new_val.is_a?(Array)
          (base_val + new_val).uniq
        else
          new_val
        end
      end
    end

    private

    def log(severity, message, ex, data, block)
      return true if level > severity
      args = block ? block.call : [message, ex, data]
      append(severity, args)
    end
  end
end
{'fraction_non_alphanumeric': 0.10635733654715275, 'fraction_numerical': 0.0, 'mean_word_length': 2.480083857442348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4374544', 'n_tokens_mistral': 1087, 'n_tokens_neox': 1065, 'n_words': 331}