<commit_before>#========================= Filebeat global options ============================

filebeat.spool_size: <%= @filebeat_config['filebeat']['spool_size'] %>
filebeat.publish_async: <%= @filebeat_config['filebeat']['publish_async'] %>
filebeat.idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %>
filebeat.registry_file: <%= @filebeat_config['filebeat']['registry_file'] %>
filebeat.config_dir: <%= @filebeat_config['filebeat']['config_dir'] %>
filebeat.shutdown_timeout: <%= @filebeat_config['filebeat']['shutdown_timeout'] %>

#================================ General =====================================

name: <%= @filebeat_config['beat_name'] %>
<%- if @filebeat_config['tags'].length > 0 -%>
tags:
<%- @filebeat_config['tags'].each do |tag| -%>
  - <%= tag %>
<%- end -%>
<%- end -%>

<%- if @filebeat_config['fields'].length > 0 -%>
fields:
  <%- @filebeat_config['fields'].each_pair do |k, v| -%>
  <%= k %>: <%= v %>
  <%- end -%>
<%- end -%>

fields_under_root: <%= @filebeat_config['fields_under_root'] %>
queue_size: <%= @filebeat_config['queue_size'] %>

# The internal queue size for bulk events in the processing pipeline.
# Do not modify this value.
#bulk_queue_size: 0

<%- if @filebeat_config['max_procs'] != :undef -%>
max_procs: <%= @filebeat_config['max_procs'] %>
<%- end -%>

#================================ Processors =====================================

# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external meta data. This section defines a list of processors
# that are applied one by one and the first one receives the initial event:
#
#   event -> filter1 -> event1 -> filter2 ->event2 ...
#
# Supported processors: drop_fields, drop_event, include_fields
#
# For example, you can use the following processors to keep
# the fields that contain CPU load percentages, but remove the fields that
# contain CPU ticks values:
#
#processors:
#- include_fields:
#    fields: ["cpu"]
#- drop_fields:
#    fields: ["cpu.user", "cpu.system"]
#
# The following example drops the events that have the HTTP response code 200:
#
#processors:
#- drop_event:
#    when:
#       equals:
#           http.code: 200
#

#================================ Outputs =====================================

<%- if @filebeat_config['output']['elasticsearch'] != nil -%>
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
  <%- if @filebeat_config['output']['elasticsearch']['enabled'] != nil -%>
  enabled: <%= @filebeat_config['output']['elasticsearch']['enabled'] %>
  <%- end -%>
  hosts:
  <%- @filebeat_config['output']['elasticsearch']['hosts'].each do |host| -%>
    - <%= host %>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['compression_level'] != nil -%>
  compression_level: <%= @filebeat_config['output']['elasticsearch']['compression_level'] %>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['protocol'] != nil -%>
  protocol: "<%= @filebeat_config['output']['elasticsearch']['protocol'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['username'] != nil -%>
  username: "<%= @filebeat_config['output']['elasticsearch']['username'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['password'] != nil -%>
  password: "<%= @filebeat_config['output']['elasticsearch']['password'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['parameters'] != nil && @filebeat_config['output']['elasticsearch']['parameters'].length > 0 -%>
  parameters:
    <%- @filebeat_config['output']['elasticsearch']['parameters'].each_pair do |k, v| -%>
    <%= k %>: <%= v %>
    <%- end -%>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['worker'] != nil -%>
  worker: <%= @filebeat_config['output']['elasticsearch']['worker'] %>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['index'] != nil -%>
  index: "<%= @filebeat_config['output']['elasticsearch']['index'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['pipeline'] != nil -%>
  pipeline: "<%= @filebeat_config['output']['elasticsearch']['pipeline'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['indices'] != nil -%>
  indices:
  <%- @filebeat_config['output']['elasticsearch']['indices'].each do |indice| -%>
    <%- if indice['name'] != nil -%>
    - index: "<%= indice['name'] %>"
      <%- if indice['pipeline'] != nil -%>
      pipeline: "<%= indice['pipeline'] %>"
      <%- end -%>
      when.contains:
        <%= indice['filter'] %>: "<%= indice['pattern'] %>"
    <%- end -%>
  <%- end -%>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['path'] != nil -%>
  path: "<%= @filebeat_config['output']['elasticsearch']['path'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['proxy_url'] != nil -%>
  proxy_url: "<%= @filebeat_config['output']['elasticsearch']['proxy_url'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['max_retries'] != nil -%><commit_msg>Conditional pipeline in output.elasticsearch (#98)<commit_after>#========================= Filebeat global options ============================

filebeat.spool_size: <%= @filebeat_config['filebeat']['spool_size'] %>
filebeat.publish_async: <%= @filebeat_config['filebeat']['publish_async'] %>
filebeat.idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %>
filebeat.registry_file: <%= @filebeat_config['filebeat']['registry_file'] %>
filebeat.config_dir: <%= @filebeat_config['filebeat']['config_dir'] %>
filebeat.shutdown_timeout: <%= @filebeat_config['filebeat']['shutdown_timeout'] %>

#================================ General =====================================

name: <%= @filebeat_config['beat_name'] %>
<%- if @filebeat_config['tags'].length > 0 -%>
tags:
<%- @filebeat_config['tags'].each do |tag| -%>
  - <%= tag %>
<%- end -%>
<%- end -%>

<%- if @filebeat_config['fields'].length > 0 -%>
fields:
  <%- @filebeat_config['fields'].each_pair do |k, v| -%>
  <%= k %>: <%= v %>
  <%- end -%>
<%- end -%>

fields_under_root: <%= @filebeat_config['fields_under_root'] %>
queue_size: <%= @filebeat_config['queue_size'] %>

# The internal queue size for bulk events in the processing pipeline.
# Do not modify this value.
#bulk_queue_size: 0

<%- if @filebeat_config['max_procs'] != :undef -%>
max_procs: <%= @filebeat_config['max_procs'] %>
<%- end -%>

#================================ Processors =====================================

# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external meta data. This section defines a list of processors
# that are applied one by one and the first one receives the initial event:
#
#   event -> filter1 -> event1 -> filter2 ->event2 ...
#
# Supported processors: drop_fields, drop_event, include_fields
#
# For example, you can use the following processors to keep
# the fields that contain CPU load percentages, but remove the fields that
# contain CPU ticks values:
#
#processors:
#- include_fields:
#    fields: ["cpu"]
#- drop_fields:
#    fields: ["cpu.user", "cpu.system"]
#
# The following example drops the events that have the HTTP response code 200:
#
#processors:
#- drop_event:
#    when:
#       equals:
#           http.code: 200
#

#================================ Outputs =====================================

<%- if @filebeat_config['output']['elasticsearch'] != nil -%>
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
  <%- if @filebeat_config['output']['elasticsearch']['enabled'] != nil -%>
  enabled: <%= @filebeat_config['output']['elasticsearch']['enabled'] %>
  <%- end -%>
  hosts:
  <%- @filebeat_config['output']['elasticsearch']['hosts'].each do |host| -%>
    - <%= host %>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['compression_level'] != nil -%>
  compression_level: <%= @filebeat_config['output']['elasticsearch']['compression_level'] %>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['protocol'] != nil -%>
  protocol: "<%= @filebeat_config['output']['elasticsearch']['protocol'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['username'] != nil -%>
  username: "<%= @filebeat_config['output']['elasticsearch']['username'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['password'] != nil -%>
  password: "<%= @filebeat_config['output']['elasticsearch']['password'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['parameters'] != nil && @filebeat_config['output']['elasticsearch']['parameters'].length > 0 -%>
  parameters:
    <%- @filebeat_config['output']['elasticsearch']['parameters'].each_pair do |k, v| -%>
    <%= k %>: <%= v %>
    <%- end -%>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['worker'] != nil -%>
  worker: <%= @filebeat_config['output']['elasticsearch']['worker'] %>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['index'] != nil -%>
  index: "<%= @filebeat_config['output']['elasticsearch']['index'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['pipeline'] != nil -%>
  pipeline: "<%= @filebeat_config['output']['elasticsearch']['pipeline'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['pipelines'] != nil -%>
  pipelines:
  <%- @filebeat_config['output']['elasticsearch']['pipelines'].each do |pipeline| -%>
    <%- if pipeline['name'] != nil -%>
    - pipeline: "<%= pipeline['name'] %>"
      when.equals:
        <%= pipeline['filter'] %>: "<%= pipeline['pattern'] %>"
    <%- end -%>
  <%- end -%>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['indices'] != nil -%>
  indices:
  <%- @filebeat_config['output']['elasticsearch']['indices'].each do |indice| -%>
    <%- if indice['name'] != nil -%>
    - index: "<%= indice['name'] %>"
      <%- if indice['pipeline'] != nil -%>
      pipeline: "<%= indice['pipeline'] %>"
      <%- end -%>
      when.contains:
        <%= indice['filter'] %>: "<%= indice['pattern'] %>"
    <%- end -%>
  <%- end -%>
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['path'] != nil -%>
  path: "<%= @filebeat_config['output']['elasticsearch']['path'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['proxy_url'] != nil -%>
  proxy_url: "<%= @filebeat_config['output']['elasticsearch']['proxy_url'] %>"
  <%- end -%>
  <%- if @filebeat_config['output']['elasticsearch']['max_retries'] != nil -%>
{'fraction_non_alphanumeric': 0.28016148718430195, 'fraction_numerical': 0.002816636935499014, 'mean_word_length': 4.757837837837838, 'pattern_counts': {'":': 0, '<': 164, '<?xml version=': 0, '>': 178, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4863963', 'n_tokens_mistral': 3514, 'n_tokens_neox': 3309, 'n_words': 602}