<commit_before>module Metric::Purging
  def self.purge_date(type)
    value = VMDB::Config.new("vmdb").config.fetch_path(:performance, :history, type.to_sym)
    return if value.nil?

    case value
    when Numeric
      value.days.ago.utc
    when String
      value.to_i_with_method.seconds.ago.utc
    when nil
    end
  end

  def self.purge_all_timer
    purge_realtime_timer
    purge_rollup_timer
  end

  def self.purge_rollup_timer
    purge_hourly_timer
    purge_daily_timer
  end

  def self.purge_daily_timer(ts = nil)
    ts ||= purge_date(:keep_daily_performances) || 6.months.ago.utc
    purge_timer(ts, "daily")
  end

  def self.purge_hourly_timer(ts = nil)
    ts ||= purge_date(:keep_hourly_performances) || 6.months.ago.utc
    purge_timer(ts, "hourly")
  end

  def self.purge_realtime_timer(ts = nil)
    ts ||= purge_date(:keep_realtime_performances) || 4.hours.ago.utc
    purge_timer(ts, "realtime")
  end

  def self.purge_timer(ts, interval)
    MiqQueue.put_unless_exists(
      :class_name    => name,
      :method_name   => "purge",
      :role          => "ems_metrics_processor",
      :queue_name    => "ems_metrics_processor",
      :state         => ["ready", "dequeue"],
      :args_selector => ->(args) { args.kind_of?(Array) && args.last == interval }
    ) do |_msg, find_options|
      find_options.merge(:args => [ts, interval])
    end
  end

  def self.purge_count(older_than, interval)
    klass, conditions = case interval
                        when 'realtime' then [Metric,       {}]
                        else             [MetricRollup, {:capture_interval_name => interval}]
                        end

    oldest = klass.select(:timestamp).where(conditions).order(:timestamp).first
    oldest = oldest.nil? ? older_than : oldest.timestamp

    klass.where(conditions).where(:timestamp => oldest..older_than).count
  end

  def self.purge(older_than, interval, window = nil, limit = nil)
    _log.info("Purging #{limit.nil? ? "all" : limit} #{interval} metrics older than [#{older_than}]...")

    klass, conditions = case interval
                        when 'realtime' then [Metric,       {}]
                        else             [MetricRollup, {:capture_interval_name => interval}]
                        end

    total = 0
    total_tag_values = 0
    _, timings = Benchmark.realtime_block(:total_time) do
      window ||= (VMDB::Config.new("vmdb").config.fetch_path(:performance, :history, :purge_window_size) || 1000)

      oldest = nil
      Benchmark.realtime_block(:query_oldest) do
        oldest = klass.select(:timestamp).where(conditions).order(:timestamp).first
        oldest = oldest.nil? ? older_than : oldest.timestamp
      end

      loop do
        batch, = Benchmark.realtime_block(:query_batch) do
          klass.select(:id).where(conditions).where(:timestamp => (oldest..older_than)).limit(window)<commit_msg>extract common klass/condition for interval<commit_after>module Metric::Purging
  def self.purge_date(type)
    value = VMDB::Config.new("vmdb").config.fetch_path(:performance, :history, type.to_sym)
    return if value.nil?

    case value
    when Numeric
      value.days.ago.utc
    when String
      value.to_i_with_method.seconds.ago.utc
    when nil
    end
  end

  def self.purge_all_timer
    purge_realtime_timer
    purge_rollup_timer
  end

  def self.purge_rollup_timer
    purge_hourly_timer
    purge_daily_timer
  end

  def self.purge_daily_timer(ts = nil)
    ts ||= purge_date(:keep_daily_performances) || 6.months.ago.utc
    purge_timer(ts, "daily")
  end

  def self.purge_hourly_timer(ts = nil)
    ts ||= purge_date(:keep_hourly_performances) || 6.months.ago.utc
    purge_timer(ts, "hourly")
  end

  def self.purge_realtime_timer(ts = nil)
    ts ||= purge_date(:keep_realtime_performances) || 4.hours.ago.utc
    purge_timer(ts, "realtime")
  end

  def self.purge_timer(ts, interval)
    MiqQueue.put_unless_exists(
      :class_name    => name,
      :method_name   => "purge",
      :role          => "ems_metrics_processor",
      :queue_name    => "ems_metrics_processor",
      :state         => ["ready", "dequeue"],
      :args_selector => ->(args) { args.kind_of?(Array) && args.last == interval }
    ) do |_msg, find_options|
      find_options.merge(:args => [ts, interval])
    end
  end

  def self.klass_for_interval(interval)
    klass, conditions = case interval
                        when 'realtime' then [Metric,       {}]
                        else             [MetricRollup, {:capture_interval_name => interval}]
                        end
  end

  def self.purge_count(older_than, interval)
    klass, conditions = klass_for_interval(interval)

    oldest = klass.select(:timestamp).where(conditions).order(:timestamp).first
    oldest = oldest.nil? ? older_than : oldest.timestamp

    klass.where(conditions).where(:timestamp => oldest..older_than).count
  end

  def self.purge(older_than, interval, window = nil, limit = nil)
    _log.info("Purging #{limit.nil? ? "all" : limit} #{interval} metrics older than [#{older_than}]...")

    klass, conditions = klass_for_interval(interval)

    total = 0
    total_tag_values = 0
    _, timings = Benchmark.realtime_block(:total_time) do
      window ||= (VMDB::Config.new("vmdb").config.fetch_path(:performance, :history, :purge_window_size) || 1000)

      oldest = nil
      Benchmark.realtime_block(:query_oldest) do
        oldest = klass.select(:timestamp).where(conditions).order(:timestamp).first
        oldest = oldest.nil? ? older_than : oldest.timestamp
      end

      loop do
        batch, = Benchmark.realtime_block(:query_batch) do
          klass.select(:id).where(conditions).where(:timestamp => (oldest..older_than)).limit(window)
{'fraction_non_alphanumeric': 0.12050078247261346, 'fraction_numerical': 0.003129890453834116, 'mean_word_length': 2.894380501015572, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3488490', 'n_tokens_mistral': 1907, 'n_tokens_neox': 1839, 'n_words': 343}