<commit_before>#encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "elasticsearch"
require "stud/buffer"

class LogStash::Outputs::Percolator < LogStash::Outputs::Base

  include Stud::Buffer

  config_name "percolator"
  milestone 1

  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
  config :pattern_index, :validate => :string, :default => "logstash-patterns"
  config :host, :validate => :string
  config :port, :validate => :string, :default => "9200"
  config :redis_host, :validate => :array, :default => ["127.0.0.1"]
  config :redis_port, :validate => :number, :default => 6379
  config :redis_socket, :validate => :string
  config :db, :validate => :number, :default => 0
  config :password, :validate => :password
  config :key, :validate => :string, :required => false, :default => "percolator"
  config :data_type, :validate => [ "list", "channel" ], :required => false, :default => "list"
  config :reconnect_interval, :validate => :number, :default => 1
  config :timeout, :validate => :number, :default => 5
  config :congestion_threshold, :validate => :number, :default => 0
  config :congestion_interval, :validate => :number, :default => 1

  public
  def register
    require 'redis'
    @logger.debug("Registered percolator plugin")
    @redis = nil
  end # def register

  public
  def receive(event)
    return unless output?(event)

    begin
      index = event.sprintf(@index)
      document_id = event["logstash_checksum"]
      message = event["message"]
      key = event["key"]
      timestamp = event["toplog_timestamp"]
      @logger.debug("Percolator args are ", :host => @host, :redis_url => @redis_url, :port => @port, :redis_socket => @redis_socket, :index => index, :document_id => document_id, :percolate_index => pattern_index)
      es_client = Elasticsearch::Client.new host: @host, port: @port
      get_index = es_client.indices.exists index: pattern_index
      unless get_index
        es_client.indices.create index: pattern_index
      end

      percolation = es_client.percolate index: pattern_index, type: 'automatic', body: { doc: { message: message } }
      alert_ids = []
      unless percolation['matches'].empty?
        percolation['matches'].each do | match |
          @logger.debug("#{document_id} matched #{match['_id']}")
          alert_ids << match['_id']
        end
        payload = {
          'alert_ids' => alert_ids,
          'document_id' => document_id,
          'access_token' => key,
          'toplog_timestamp' => timestamp
        }

        send_to_redis(payload.to_json)

      end
      event.remove("logstash_checksum")
      event.remove("random")
    rescue Exception => e
      @logger.debug(e.message)<commit_msg>removing hardcodes<commit_after>#encoding: utf-8
require "logstash/outputs/base"
require "logstash/namespace"
require "elasticsearch"
require "stud/buffer"

class LogStash::Outputs::Percolator < LogStash::Outputs::Base

  include Stud::Buffer

  config_name "percolator"
  milestone 1

  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
  config :pattern_index, :validate => :string, :default => "logstash-patterns"
  config :host, :validate => :string
  config :port, :validate => :string, :default => "9200"
  config :redis_host, :validate => :array, :default => ["127.0.0.1"]
  config :redis_port, :validate => :number, :default => 6379
  config :redis_socket, :validate => :string
  config :db, :validate => :number, :default => 0
  config :password, :validate => :password
  config :key, :validate => :string, :required => false, :default => "percolator"
  config :data_type, :validate => [ "list", "channel" ], :required => false, :default => "list"
  config :reconnect_interval, :validate => :number, :default => 1
  config :timeout, :validate => :number, :default => 5
  config :congestion_threshold, :validate => :number, :default => 0
  config :congestion_interval, :validate => :number, :default => 1
  config :document_id, :validate => :string, :required => true

  public
  def register
    require 'redis'
    @logger.debug("Registered percolator plugin")
    @redis = nil
  end # def register

  public
  def receive(event)
    return unless output?(event)

    begin
      index = event.sprintf(@index)
      document_id = event.sprintf(@document_id)
      message = event["message"]
      key = event["key"]
      timestamp = event["toplog_timestamp"]
      @logger.debug("Percolator args are ", :host => @host, :redis_url => @redis_url, :port => @port, :redis_socket => @redis_socket, :index => index, :document_id => document_id, :percolate_index => pattern_index)
      es_client = Elasticsearch::Client.new host: @host, port: @port
      get_index = es_client.indices.exists index: pattern_index
      unless get_index
        es_client.indices.create index: pattern_index
      end

      percolation = es_client.percolate index: pattern_index, type: 'automatic', body: { doc: { message: message } }
      alert_ids = []
      unless percolation['matches'].empty?
        percolation['matches'].each do | match |
          @logger.debug("#{document_id} matched #{match['_id']}")
          alert_ids << match['_id']
        end
        payload = {
          'matches' => matches,
          'document_id' => document_id
        }

        send_to_redis(payload.to_json)

      end
      event.remove("logstash_checksum")
      event.remove("random")
    rescue Exception => e
      @logger.debug(e.message)
{'fraction_non_alphanumeric': 0.1443336355394379, 'fraction_numerical': 0.007615593834995467, 'mean_word_length': 3.6864910790144436, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3596476', 'n_tokens_mistral': 1740, 'n_tokens_neox': 1697, 'n_words': 422}