<commit_before># DNS Filter
#
# This filter will resolve any IP addresses from a field of your choosing.
#

require "logstash/filters/base"
require "logstash/namespace"

# The DNS filter performs a lookup (either an A record/CNAME record lookup
# or a reverse lookup at the PTR record) on records specified under the
# "reverse" and "resolve" arrays.
#
# The config should look like this:
#
#     filter {
#       dns {
#         type => 'type'
#         reverse => [ "@source_host", "field_with_address" ]
#         resolve => [ "field_with_fqdn" ]
#         action => "replace"
#       }
#     }
#
# Caveats: at the moment, there's no way to tune the timeout with the 'resolv'
# core library.  It does seem to be fixed in here:
#
#   http://redmine.ruby-lang.org/issues/5100
#
# but isn't currently in JRuby.
class LogStash::Filters::DNS < LogStash::Filters::Base

  config_name "dns"

  # Reverse resolve one or more fields.
  config :reverse, :validate => :array

  # Forward resolve one or more fields.
  config :resolve, :validate => :array

  # Determine what action to do: append or replace the values in the fields
  # specified under "reverse" and "resolve."
  config :action, :validate => [ "append", "replace" ], :require => true

  public
  def register
    require "resolv"

    @ip_validator = Resolv::AddressRegex
  end # def register

  public
  def filter(event)
    return unless event.type == @type or @type.nil?

    resolve(event) if @resolve
    reverse(event) if @reverse

    filter_matched(event)
  end

  private
  def resolve(event)
    @resolve.each do |field|
      begin
        address = Resolv.getaddress(event[field])
      rescue Resolv::ResolvError
        @logger.debug("DNS: couldn't resolve the hostname.")
        return
      rescue Resolv::ResolvTimeout
        @logger.debug("DNS: timeout on resolving the hostname.")
        return
      rescue SocketError
        @logger.debug("DNS: Encountered SocketError: name or service not known.")
        return
      end
      if @action == "replace"
        event[field] = address
      else
        event[field] << address<commit_msg>Add fields to debug messages, so we know what's failing the lookup. Also output error on SocketError, and show what's causing it and what the exact SocketError message is.<commit_after># DNS Filter
#
# This filter will resolve any IP addresses from a field of your choosing.
#

require "logstash/filters/base"
require "logstash/namespace"

# The DNS filter performs a lookup (either an A record/CNAME record lookup
# or a reverse lookup at the PTR record) on records specified under the
# "reverse" and "resolve" arrays.
#
# The config should look like this:
#
#     filter {
#       dns {
#         type => 'type'
#         reverse => [ "@source_host", "field_with_address" ]
#         resolve => [ "field_with_fqdn" ]
#         action => "replace"
#       }
#     }
#
# Caveats: at the moment, there's no way to tune the timeout with the 'resolv'
# core library.  It does seem to be fixed in here:
#
#   http://redmine.ruby-lang.org/issues/5100
#
# but isn't currently in JRuby.
class LogStash::Filters::DNS < LogStash::Filters::Base

  config_name "dns"

  # Reverse resolve one or more fields.
  config :reverse, :validate => :array

  # Forward resolve one or more fields.
  config :resolve, :validate => :array

  # Determine what action to do: append or replace the values in the fields
  # specified under "reverse" and "resolve."
  config :action, :validate => [ "append", "replace" ], :require => true

  public
  def register
    require "resolv"

    @ip_validator = Resolv::AddressRegex
  end # def register

  public
  def filter(event)
    return unless event.type == @type or @type.nil?

    resolve(event) if @resolve
    reverse(event) if @reverse

    filter_matched(event)
  end

  private
  def resolve(event)
    @resolve.each do |field|
      begin
        address = Resolv.getaddress(event[field])
      rescue Resolv::ResolvError
        @logger.debug("DNS: couldn't resolve the hostname. Field: #{event[field]}")
        return
      rescue Resolv::ResolvTimeout
        @logger.debug("DNS: timeout on resolving the hostname. Field: #{event[field]}")
        return
      rescue SocketError => e
        @logger.debug("DNS: Encountered SocketError. Field: #{event[field]}, error: #{e}")
        return
      end
      if @action == "replace"
        event[field] = address
      else
        event[field] << address
{'fraction_non_alphanumeric': 0.10483870967741936, 'fraction_numerical': 0.0017921146953405018, 'mean_word_length': 2.993738819320215, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4863418', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1327, 'n_words': 469}