<commit_before>class ImportSystemEventJob < ActiveJob::Base
  queue_as :import

  def perform(options = {})
    options.symbolize_keys!
    count = options.fetch(:count, 0)

    Syslog::Systemevent.find_each do |sysevent|
      begin
        result = CreateEventService.new(sysevent.event_attributes).call
        if result.success?
          sysevent.destroy
          if result.event.present?
            LogEvent.log(result.event) if Rails.env.development?
            ImportEventSignatureService.new.call(result.event)
          end
        else
          Rails.logger.warn(result.error_messages.join(", ")) if Rails.env.developement?
          LogSyslogsysevent.log(sysevent)
        end
      rescue => e
         Rails.logger.warn("RESCUE: #{e.inspect}")
         LogSyslogsysevent.log(sysevent)
      end
    end
    # execute complexe event_rules (base on active record statements)
    Event.assign_filters
    Event.joins(:event_rule).where(event_rules: {action: 'drop'}).count
  end
end<commit_msg>destroy events with drop rule, but only created from the current job<commit_after>class ImportSystemEventJob < ActiveJob::Base
  queue_as :import

  def perform(options = {})
    options.symbolize_keys!
    count = options.fetch(:count, 0)

    timestamp = Time.now

    Syslog::Systemevent.find_each do |sysevent|
      begin
        result = CreateEventService.new(sysevent.event_attributes).call
        if result.success?
          sysevent.destroy
          if result.event.present?
            LogEvent.log(result.event) if Rails.env.development?
            ImportEventSignatureService.new.call(result.event)
          end
        else
          Rails.logger.warn(result.error_messages.join(", ")) if Rails.env.developement?
          LogSyslogsysevent.log(sysevent)
        end
      rescue => e
         Rails.logger.warn("RESCUE: #{e.inspect}")
         LogSyslogsysevent.log(sysevent)
      end
    end
    # execute complexe event_rules (base on active record statements)
    Event.assign_filters
    # drop events with event_rule.action = drop, but only if created
    # in this job for safety reasons
    Event.joins(:event_rule).
      where("events.created_at >= :ts", ts: timestamp).
      where(event_rules: {action: 'drop'}).
      destroy_all
  end
end
{'fraction_non_alphanumeric': 0.09601052170100834, 'fraction_numerical': 0.0008768084173608067, 'mean_word_length': 2.6985413290113454, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '346954', 'n_tokens_mistral': 681, 'n_tokens_neox': 673, 'n_words': 141}