<commit_before>require "logstash/namespace"
require "logstash/logging"
require "logstash/plugin"
require "logstash/config/mixin"

class LogStash::Filters::Base < LogStash::Plugin
  include LogStash::Config::Mixin

  config_name "filter"

  # Note that all of the specified routing options (type,tags.exclude_tags,include_fields,exclude_fields)
  # must be met in order for the event to be handled by the filter.

  # The type to act on. If a type is given, then this filter will only
  # act on messages with the same type. See any input plugin's "type"
  # attribute for more.
  # Optional.
  config :type, :validate => :string, :default => ""

  # Only handle events with all/any (controlled by include_any config option) of these tags.
  # Optional.
  # TODO(piavlo): sould we rename/alias this to include_tags for clearness and consistency?
  config :tags, :validate => :array, :default => []

  # Only handle events without all/any (controlled by exclude_any config option) of these tags.
  # Optional.
  config :exclude_tags, :validate => :array, :default => []

  # Only handle events with all/any (controlled by include_any config option) of these fields.
  # Optional.
  config :include_fields, :validate => :array, :default => []

  # Only handle events without all/any (controlled by exclude_any config option) of these fields.
  # Optional.
  config :exclude_fields, :validate => :array, :default => []

  # Should all or any of the specified tags/include_fields be present for event to
  # be handled. Defaults to all.
  config :include_any, :validate => :boolean, :default => false

  # Should all or any of the specified exclude_tags/exclude_fields be missing for event to
  # be handled. Defaults to all.
  config :exclude_any, :validate => :boolean, :default => true

  # If this filter is successful, add arbitrary tags to the event.
  # Tags can be dynamic and include parts of the event using the %{field}
  # syntax. Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         add_tag => [ "foo_%{somefield}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would add a tag "foo_hello"
  config :add_tag, :validate => :array, :default => []

  # If this filter is successful, remove arbitrary tags from the event.
  # Tags can be dynamic and include parts of the event using the %{field}
  # syntax. Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         remove_tag => [ "foo_%{somefield}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would remove the tag "foo_hello" if it is present
  config :remove_tag, :validate => :array, :default => []

  # If this filter is successful, add any arbitrary fields to this event.
  # Tags can be dynamic and include parts of the event using the %{field}
  # Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         add_field => [ "foo_%{somefield}", "Hello world, from %{@source}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would add field "foo_hello" if it is present, with the
  # value above and the %{@source} piece replaced with that value from the
  # event.
  config :add_field, :validate => :hash, :default => {}

  # If this filter is successful, remove arbitrary fields from this event.
  # Fields names can be dynamic and include parts of the event using the %{field}
  # Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         remove_field => [ "foo_%{somefield}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would remove the field with name "foo_hello" if it is present
  config :remove_field, :validate => :array, :default => []

  RESERVED = ["type", "tags", "exclude_tags", "include_fields", "exclude_fields", "add_tag", "remove_tag", "add_field", "remove_field"]

  public
  def initialize(params)
    super
    config_init(params)
    @threadsafe = true
  end # def initialize

  public
  def register
    raise "#{self.class}#register must be overidden"
  end # def register

  public
  def filter(event)
    raise "#{self.class}#filter must be overidden"
  end # def filter

  public
  def execute(event, &block)
    filter(event, &block)
  end # def execute

  public
  def threadsafe?
    @threadsafe<commit_msg>need to add include_any & exclude_any as reserved words<commit_after>require "logstash/namespace"
require "logstash/logging"
require "logstash/plugin"
require "logstash/config/mixin"

class LogStash::Filters::Base < LogStash::Plugin
  include LogStash::Config::Mixin

  config_name "filter"

  # Note that all of the specified routing options (type,tags.exclude_tags,include_fields,exclude_fields)
  # must be met in order for the event to be handled by the filter.

  # The type to act on. If a type is given, then this filter will only
  # act on messages with the same type. See any input plugin's "type"
  # attribute for more.
  # Optional.
  config :type, :validate => :string, :default => ""

  # Only handle events with all/any (controlled by include_any config option) of these tags.
  # Optional.
  # TODO(piavlo): sould we rename/alias this to include_tags for clearness and consistency?
  config :tags, :validate => :array, :default => []

  # Only handle events without all/any (controlled by exclude_any config option) of these tags.
  # Optional.
  config :exclude_tags, :validate => :array, :default => []

  # Only handle events with all/any (controlled by include_any config option) of these fields.
  # Optional.
  config :include_fields, :validate => :array, :default => []

  # Only handle events without all/any (controlled by exclude_any config option) of these fields.
  # Optional.
  config :exclude_fields, :validate => :array, :default => []

  # Should all or any of the specified tags/include_fields be present for event to
  # be handled. Defaults to all.
  config :include_any, :validate => :boolean, :default => false

  # Should all or any of the specified exclude_tags/exclude_fields be missing for event to
  # be handled. Defaults to all.
  config :exclude_any, :validate => :boolean, :default => true

  # If this filter is successful, add arbitrary tags to the event.
  # Tags can be dynamic and include parts of the event using the %{field}
  # syntax. Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         add_tag => [ "foo_%{somefield}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would add a tag "foo_hello"
  config :add_tag, :validate => :array, :default => []

  # If this filter is successful, remove arbitrary tags from the event.
  # Tags can be dynamic and include parts of the event using the %{field}
  # syntax. Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         remove_tag => [ "foo_%{somefield}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would remove the tag "foo_hello" if it is present
  config :remove_tag, :validate => :array, :default => []

  # If this filter is successful, add any arbitrary fields to this event.
  # Tags can be dynamic and include parts of the event using the %{field}
  # Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         add_field => [ "foo_%{somefield}", "Hello world, from %{@source}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would add field "foo_hello" if it is present, with the
  # value above and the %{@source} piece replaced with that value from the
  # event.
  config :add_field, :validate => :hash, :default => {}

  # If this filter is successful, remove arbitrary fields from this event.
  # Fields names can be dynamic and include parts of the event using the %{field}
  # Example:
  #
  #     filter {
  #       %PLUGIN% {
  #         remove_field => [ "foo_%{somefield}" ]
  #       }
  #     }
  #
  # If the event has field "somefield" == "hello" this filter, on success,
  # would remove the field with name "foo_hello" if it is present
  config :remove_field, :validate => :array, :default => []

  RESERVED = ["type", "tags", "exclude_tags", "include_fields", "exclude_fields", "add_tag", "remove_tag", "add_field", "remove_field", "include_any", "exclude_any"]

  public
  def initialize(params)
    super
    config_init(params)
    @threadsafe = true
  end # def initialize

  public
  def register
    raise "#{self.class}#register must be overidden"
  end # def register

  public
  def filter(event)
    raise "#{self.class}#filter must be overidden"
  end # def filter

  public
  def execute(event, &block)
    filter(event, &block)
  end # def execute

  public
  def threadsafe?
    @threadsafe
{'fraction_non_alphanumeric': 0.10922937031143441, 'fraction_numerical': 0.0, 'mean_word_length': 3.244573082489146, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1180750', 'n_tokens_mistral': 2720, 'n_tokens_neox': 2640, 'n_words': 1008}