<commit_before>require 'util/postgres_admin'

module MiqServer::LogManagement
  extend ActiveSupport::Concern

  included do
    belongs_to :log_file_depot, :class_name => "FileDepot"
    has_many   :log_files, :dependent => :destroy, :as => :resource
  end

  def format_log_time(time)
    time.respond_to?(:strftime) ? time.strftime("%Y%m%d_%H%M%S") : "unknown"
  end

  def post_historical_logs(taskid, log_depot)
    task = MiqTask.find(taskid)
    resource = who_am_i

    # Post all compressed logs for a specific date + configs, creating a new row per day
    VMDB::Util.compressed_log_patterns.each do |pattern|
      evm = VMDB::Util.get_evm_log_for_date(pattern)
      next if evm.nil?

      log_start, log_end = VMDB::Util.get_log_start_end_times(evm)
      date = File.basename(pattern).gsub!(/\*|\.gz/, "")

      date_string = "#{format_log_time(log_start)} #{format_log_time(log_end)}" unless log_start.nil? && log_end.nil?
      date_string ||= date
      name = "Archived #{self.name} logs #{date_string} "
      desc = "Logs for Zone #{zone.name rescue nil} Server #{self.name} #{date_string}"

      cond = {:historical => true, :name => name, :state => 'available'}
      cond[:logging_started_on] = log_start unless log_start.nil?
      cond[:logging_ended_on] = log_end unless log_end.nil?
      logfile = log_files.find_by(cond)
      if logfile && logfile.log_uri.nil?
        _log.info("Historical logfile already exists with id: [#{logfile.id}] for [#{resource}] dated: [#{date}] with contents from: [#{log_start}] to: [#{log_end}]")
        next
      else
        logfile = LogFile.historical_logfile
      end

      msg = "Creating historical Logfile for [#{resource}] dated: [#{date}] from: [#{log_start}] to [#{log_end}]"
      task.update_status("Active", "Ok", msg)
      _log.info(msg)

      begin
        log_files << logfile
        save
        msg = "Zipping and posting historical logs and configs on #{resource}"
        task.update_status("Active", "Ok", msg)
        _log.info(msg)

        patterns = [pattern]
        cfg_pattern = ::Settings.log.collection.archive.pattern
        patterns += cfg_pattern if cfg_pattern.kind_of?(Array)

        local_file = VMDB::Util.zip_logs("evm_server_daily.zip", patterns, "admin")

        logfile.update_attributes(
          :file_depot         => log_depot,
          :local_file         => local_file,
          :logging_started_on => log_start,
          :logging_ended_on   => log_end,
          :name               => name,
          :description        => desc,
          :miq_task_id        => task.id
        )

        logfile.upload
      rescue StandardError, Timeout::Error => err
        logfile.update_attributes(:state => "error")
        _log.error("Posting of historical logs failed for #{resource} due to error: [#{err.class.name}] [#{err}]")
        raise
      end

      msg = "Historical log files from #{resource} for #{date} are posted"
      task.update_status("Active", "Ok", msg)
      _log.info(msg)

      # TODO: If the gz has been posted and the gz is more than X days old, delete it
    end
  end

  def _post_my_logs(options)
    # Make the request to the MiqServer whose logs are needed
    MiqQueue.put_or_update(
      :class_name  => self.class.name,
      :instance_id => id,
      :method_name => "post_logs",
      :server_guid => guid,
      :zone        => my_zone,
    ) do |msg, item|
      _log.info("Previous adhoc log collection is still running, skipping...Resource: [#{self.class.name}], id: [#{id}]") unless msg.nil?
      item.merge(
        :miq_callback => options.delete(:callback),
        :msg_timeout  => options.delete(:timeout),
        :priority     => MiqQueue::HIGH_PRIORITY,
        :args         => [options])
    end
  end

  def synchronize_logs(*args)
    options = args.extract_options!<commit_msg>Logs: Convert put_or_update to put_unless_exists<commit_after>require 'util/postgres_admin'

module MiqServer::LogManagement
  extend ActiveSupport::Concern

  included do
    belongs_to :log_file_depot, :class_name => "FileDepot"
    has_many   :log_files, :dependent => :destroy, :as => :resource
  end

  def format_log_time(time)
    time.respond_to?(:strftime) ? time.strftime("%Y%m%d_%H%M%S") : "unknown"
  end

  def post_historical_logs(taskid, log_depot)
    task = MiqTask.find(taskid)
    resource = who_am_i

    # Post all compressed logs for a specific date + configs, creating a new row per day
    VMDB::Util.compressed_log_patterns.each do |pattern|
      evm = VMDB::Util.get_evm_log_for_date(pattern)
      next if evm.nil?

      log_start, log_end = VMDB::Util.get_log_start_end_times(evm)
      date = File.basename(pattern).gsub!(/\*|\.gz/, "")

      date_string = "#{format_log_time(log_start)} #{format_log_time(log_end)}" unless log_start.nil? && log_end.nil?
      date_string ||= date
      name = "Archived #{self.name} logs #{date_string} "
      desc = "Logs for Zone #{zone.name rescue nil} Server #{self.name} #{date_string}"

      cond = {:historical => true, :name => name, :state => 'available'}
      cond[:logging_started_on] = log_start unless log_start.nil?
      cond[:logging_ended_on] = log_end unless log_end.nil?
      logfile = log_files.find_by(cond)
      if logfile && logfile.log_uri.nil?
        _log.info("Historical logfile already exists with id: [#{logfile.id}] for [#{resource}] dated: [#{date}] with contents from: [#{log_start}] to: [#{log_end}]")
        next
      else
        logfile = LogFile.historical_logfile
      end

      msg = "Creating historical Logfile for [#{resource}] dated: [#{date}] from: [#{log_start}] to [#{log_end}]"
      task.update_status("Active", "Ok", msg)
      _log.info(msg)

      begin
        log_files << logfile
        save
        msg = "Zipping and posting historical logs and configs on #{resource}"
        task.update_status("Active", "Ok", msg)
        _log.info(msg)

        patterns = [pattern]
        cfg_pattern = ::Settings.log.collection.archive.pattern
        patterns += cfg_pattern if cfg_pattern.kind_of?(Array)

        local_file = VMDB::Util.zip_logs("evm_server_daily.zip", patterns, "admin")

        logfile.update_attributes(
          :file_depot         => log_depot,
          :local_file         => local_file,
          :logging_started_on => log_start,
          :logging_ended_on   => log_end,
          :name               => name,
          :description        => desc,
          :miq_task_id        => task.id
        )

        logfile.upload
      rescue StandardError, Timeout::Error => err
        logfile.update_attributes(:state => "error")
        _log.error("Posting of historical logs failed for #{resource} due to error: [#{err.class.name}] [#{err}]")
        raise
      end

      msg = "Historical log files from #{resource} for #{date} are posted"
      task.update_status("Active", "Ok", msg)
      _log.info(msg)

      # TODO: If the gz has been posted and the gz is more than X days old, delete it
    end
  end

  def _post_my_logs(options)
    # Make the request to the MiqServer whose logs are needed
    MiqQueue.create_with(
      :miq_callback => options.delete(:callback),
      :msg_timeout  => options.delete(:timeout),
      :priority     => MiqQueue::HIGH_PRIORITY,
      :args         => [options]
    ).put_unless_exists(
      :class_name  => self.class.name,
      :instance_id => id,
      :method_name => "post_logs",
      :server_guid => guid,
      :zone        => my_zone,
    ) do |msg|
      _log.info("Previous adhoc log collection is still running, skipping...Resource: [#{self.class.name}], id: [#{id}]") unless msg.nil?
      nil
    end
  end

  def synchronize_logs(*args)
    options = args.extract_options!
{'fraction_non_alphanumeric': 0.12675874532076933, 'fraction_numerical': 0.0, 'mean_word_length': 2.822397631968426, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4611952', 'n_tokens_mistral': 2523, 'n_tokens_neox': 2474, 'n_words': 593}