<commit_before>require 'logger'
require 'ostruct'
require_relative 'null_observer'
require 'processor/logger_messages'

module Processor
  module Observer
    class Logger < NullObserver
      def initialize(logger = nil, options = {})
        @logger_source = logger
        @messages = options.fetch :messages, nil
        @messages = OpenStruct.new @messages if @messages.is_a? Hash

        super options
      end

      def processing_started(processor)
        initialize_logger(processor)
        logger.info "Processing of #{processor.name} started."
        messenger.info messages.initialized
      end

      def before_record_processing(record)
        logger.debug "Record #{id_for record} is going to be processed"
      end

      def after_record_processing(record, result)
        logger.info "Processed #{id_for record}: #{result}"
      end

      def processing_finalized(processor)
        logger.info "Processing of #{processor.name} finished."
        messenger.message messages.finished
      end

      def record_processing_error(record, exception)
        logger.error "Error processing #{id_for record}: #{exception}"
      end

      def processing_error(processor, exception)
        logger.fatal "Processing #{processor.name} FAILED: #{exception.backtrace}"
      end

      private
      attr_reader :logger, :messages

      def initialize_logger(processor)
        @logger =
          if @logger_source.is_a? Proc
            @logger_source.call processor.name
          else
            @logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
              logger.level = ::Logger::INFO
            end
          end
        messenger.debug "Observer initialized with logger #{@logger}"

        @messages ||= Processor::LoggerMessages.new logger
      end

      def create_log_filename(processor_name)
        FileUtils.mkdir log_directory unless File.directory? log_directory
        @log_file_name = "#{log_directory}/#{processor_name}_on_#{current_time_string}.log"
      end

      def log_directory
        "log"
      end

      def current_time_string
        Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
      end

      def id_for record
        [:uid, :id, :to_token, :token, :to_sym].each do |method|
          return record.public_send method if record.respond_to? method
        end

        [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
          return record[method] if record.key? method<commit_msg>Add warn message if creating a new directory<commit_after>require 'logger'
require 'ostruct'
require_relative 'null_observer'
require 'processor/logger_messages'

module Processor
  module Observer
    class Logger < NullObserver
      def initialize(logger = nil, options = {})
        @logger_source = logger
        @messages = options.fetch :messages, nil
        @messages = OpenStruct.new @messages if @messages.is_a? Hash

        super options
      end

      def processing_started(processor)
        initialize_logger(processor)
        logger.info "Processing of #{processor.name} started."
        messenger.info messages.initialized
      end

      def before_record_processing(record)
        logger.debug "Record #{id_for record} is going to be processed"
      end

      def after_record_processing(record, result)
        logger.info "Processed #{id_for record}: #{result}"
      end

      def processing_finalized(processor)
        logger.info "Processing of #{processor.name} finished."
        messenger.message messages.finished
      end

      def record_processing_error(record, exception)
        logger.error "Error processing #{id_for record}: #{exception}"
      end

      def processing_error(processor, exception)
        logger.fatal "Processing #{processor.name} FAILED: #{exception.backtrace}"
      end

      private
      attr_reader :logger, :messages

      def initialize_logger(processor)
        @logger =
          if @logger_source.is_a? Proc
            @logger_source.call processor.name
          else
            @logger_source or ::Logger.new(create_log_filename(processor.name)).tap do |logger|
              logger.level = ::Logger::INFO
            end
          end
        messenger.debug "Observer initialized with logger #{@logger}"

        @messages ||= Processor::LoggerMessages.new logger
      end

      def create_log_filename(processor_name)
        unless File.directory? log_directory
          FileUtils.mkdir log_directory
          messenger.warn "Created new directory for logs: #{File.absolute_path log_directory}"
        end

        @log_file_name = "#{log_directory}/#{processor_name}_on_#{current_time_string}.log"
      end

      def log_directory
        "log"
      end

      def current_time_string
        Time.now.gmtime.strftime "%Y-%m-%d_%H%M%S_UTC"
      end

      def id_for record
        [:uid, :id, :to_token, :token, :to_sym].each do |method|
          return record.public_send method if record.respond_to? method
        end

        [:uid, :id, :token, :sym, :UID, :ID, :TOKEN, :SYM].each do |method|
          return record[method] if record.key? method
{'fraction_non_alphanumeric': 0.0906801007556675, 'fraction_numerical': 0.0, 'mean_word_length': 2.630098452883263, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '797049', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1383, 'n_words': 371}