<commit_before>require "logstash/filters/base"
require "logstash/namespace"

gem "jls-grok", ">=0.3.3209"
require "grok" # rubygem 'jls-grok'

class LogStash::Filters::Grok < LogStash::Filters::Base

  config_name "grok"
  config :pattern => nil
  config :patterns_dir => nil
  config :drop_if_match => :boolean  # googlecode/issue/26

  @@grokpiles = Hash.new { |h, k| h[k] = [] }
  @@grokpiles_lock = Mutex.new

  public
  def initialize(params)
    super
  end # def initialize

  public
  def register
    # TODO(2.0): support grok pattern discovery
    @patterns_dir ||= "#{File.dirname(__FILE__)}/../../../patterns/*"
    @pile = Grok::Pile.new
    Dir.glob(@patterns_dir).each do |path|
      @pile.add_patterns_from_file(path)
    end

    @pattern.each do |pattern|
      groks = @pile.compile(pattern)
      @logger.debug(["Compiled pattern", pattern, groks[-1].expanded_pattern])
    end

    @@grokpiles_lock.synchronize do
      @@grokpiles[@type] << @pile
    end
  end # def register

  public
  def filter(event)
    # parse it with grok
    message = event.message
    match = false

    @logger.debug(["Running grok filter", event])
    @@grokpiles[event.type].each do |pile|
      grok, match = @pile.match(message)
      break if match
    end

    if match
      match.each_capture do |key, value|
        if key.include?(":")
          key = key.split(":")[1]
        end
        if event.message == value
          # Skip patterns that match the entire line
          @logger.debug("Skipping capture '#{key}' since it matches the whole line.")
          next
        end

        if event.fields[key].is_a?(String)
          event.fields[key] = [event.fields[key]]
        elsif event.fields[key] == nil
          event.fields[key] = []
        end

        if value && !value.empty?
          event.fields[key] << value
        end
      end
      filter_matched(event)
    else
      # Tag this event if we can't parse it. We can use this later to
      # reparse+reindex logs if we improve the patterns given .
      event.tags << "_grokparsefailure"
    end<commit_msg>- Allow type coercion of named captures from grok.   Syntax: %{FOO:bar:type}   Type can be 'int' or 'float' - otherwise is assumed string.   The result is that the value captured is coerced to the requested type   in the logstash event for later processing.<commit_after>require "logstash/filters/base"
require "logstash/namespace"

gem "jls-grok", ">=0.3.3209"
require "grok" # rubygem 'jls-grok'

class LogStash::Filters::Grok < LogStash::Filters::Base

  config_name "grok"
  config :pattern => nil
  config :patterns_dir => nil
  config :drop_if_match => :boolean  # googlecode/issue/26

  @@grokpiles = Hash.new { |h, k| h[k] = [] }
  @@grokpiles_lock = Mutex.new

  public
  def initialize(params)
    super
  end # def initialize

  public
  def register
    # TODO(2.0): support grok pattern discovery
    @patterns_dir ||= "#{File.dirname(__FILE__)}/../../../patterns/*"
    @pile = Grok::Pile.new
    Dir.glob(@patterns_dir).each do |path|
      @pile.add_patterns_from_file(path)
    end

    @pattern.each do |pattern|
      groks = @pile.compile(pattern)
      @logger.debug(["Compiled pattern", pattern, groks[-1].expanded_pattern])
    end

    @@grokpiles_lock.synchronize do
      @@grokpiles[@type] << @pile
    end
  end # def register

  public
  def filter(event)
    # parse it with grok
    message = event.message
    match = false

    @logger.debug(["Running grok filter", event])
    @@grokpiles[event.type].each do |pile|
      grok, match = @pile.match(message)
      break if match
    end

    if match
      match.each_capture do |key, value|
        match_type = nil
        if key.include?(":")
          name, key, match_type = key.split(":")
        end

        # http://code.google.com/p/logstash/issues/detail?id=45
        # Permit typing of captures by giving an additional colon and a type,
        # like: %{FOO:name:int} for int coercion.
        case match_type
          when "int"
            value = value.to_i
          when "float"
            value = value.to_f
        end

        if event.message == value
          # Skip patterns that match the entire line
          @logger.debug("Skipping capture '#{key}' since it matches the whole line.")
          next
        end

        if event.fields[key].is_a?(String)
          event.fields[key] = [event.fields[key]]
        elsif event.fields[key] == nil
          event.fields[key] = []
        end

        # If value is not nil, or responds to empty and is not empty, add the
        # value to the event.
        if !value.nil? && (!value.empty? rescue true)
          event.fields[key] << value
        end
      end
      filter_matched(event)
    else
      # Tag this event if we can't parse it. We can use this later to
      # reparse+reindex logs if we improve the patterns given .
      event.tags << "_grokparsefailure"
    end
{'fraction_non_alphanumeric': 0.11902834008097166, 'fraction_numerical': 0.005060728744939271, 'mean_word_length': 2.4408077994428967, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6585184', 'n_tokens_mistral': 1628, 'n_tokens_neox': 1559, 'n_words': 443}