<commit_before><% is_v1 = version.split("-").first.start_with?('v1') %>
<% target = (dockerfile.split("/").last.split("-").last) %>
<% is_alpine = (dockerfile.split("/").last.split("-").first == "alpine") %>

# AUTOMATICALLY GENERATED
# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb

@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
@include kubernetes.conf
@include conf.d/*.conf

<% case target when "elasticsearch"%>
<match **>
   @type elasticsearch
   @id out_es
   @log_level info
   include_tag_key true
   host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
   port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
   path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
   scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
   ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
   ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
   user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
   password "#{ENV['<PASSWORD>']}"
   reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}"
   reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}"
   reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}"
   log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}"
   logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
   logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}"
   index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}"
   type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}"
<% if is_v1 %>
   <buffer>
     flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
     flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
     chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
     queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
     retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
     retry_forever true
   </buffer>
<% else %>
   buffer_chunk_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
   buffer_queue_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
   flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
   max_retry_wait "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
   disable_retry_limit
   num_threads "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
<% end %>
</match>
<%  when "logentries"%>
<match **>
   @type logentries
   @id out_logentries
   use_json true
   tag_access_log stdout
   tag_error_log stderr
   config_path /etc/logentries/tokens.yaml
</match>
<%  when "loggly"%>
<match **>
   @type loggly
   @id out_loggly
   @log_level info
   loggly_url "https://logs-01.loggly.com/bulk/#{ENV['LOGGLY_TOKEN']}/tag/#{ENV['LOGGLY_TAGS'] || 'fluentd'}/bulk"
</match>
<%  when "cloudwatch"%>
<match **>
  @type cloudwatch_logs
  @id out_cloudwatch_logs
  log_group_name "#{ENV['LOG_GROUP_NAME']}"
  auto_create_stream true
  use_tag_as_stream true
<% if is_v1 %>
  json_handler yajl # To avoid UndefinedConversionError
  log_rejected_request "#{ENV['LOG_REJECTED_REQUEST']}" # Log rejected request for missing parts
<% end %>
</match>
<%  when "s3"%>
<match **>
  # docs: https://docs.fluentd.org/v0.12/articles/out_s3
  # note: this configuration relies on the nodes have an IAM instance profile with access to your S3 bucket
  @type s3
  @id out_s3
  @log_level info
  s3_bucket "#{ENV['S3_BUCKET_NAME']}"
  s3_region "#{ENV['S3_BUCKET_REGION']}"
<% if is_v1 %>
  s3_object_key_format %{path}%Y/%m/%d/cluster-log-%{index}.%{file_extension}
  <inject>
    time_key time
    tag_key tag
    localtime false
  </inject>
  <buffer>
    @type file
    path /var/log/fluentd-buffers/s3.buffer
    timekey 3600
    timekey_use_utc true
    chunk_limit_size 256m
  </buffer>
<% else %>
  s3_object_key_format %{path}%{time_slice}/cluster-log-%{index}.%{file_extension}
  time_slice_format %Y/%m/%d
  time_slice_wait 10m
  utc
  include_time_key true
  include_tag_key true
  buffer_chunk_limit 256m
  buffer_path /var/log/fluentd-buffers/s3.buffer
<% end %>
</match>
<%  when "gcs"%>
<match **>
  # docs: https://github.com/daichirata/fluent-plugin-gcs
  # this configuration relies on the nodes having permission to write on your gs bucket
  @type gcs
  @id out_gcs
  project "#{ENV['GCS_BUCKET_PROJECT']}"
  bucket "#{ENV['GCS_BUCKET_NAME']}"
  object_key_format %{path}%{time_slice}/%{hostname}/%{index}.%{file_extension}
  path logs/
  buffer_path /var/log/fluentd-buffers/gcs.buffer
  time_slice_format %Y/%m/%d
  time_slice_wait 10m
  utc
</match>
<% when "stackdriver"%>
<match **>
  @type google_cloud
  @id out_google_cloud

  # Set the buffer type to file to improve the reliability and reduce the memory consumption
  buffer_type file
  buffer_path /var/log/fluentd-buffers/stackdriver.buffer
  # Set queue_full action to block because we want to pause gracefully
  # in case of the off-the-limits load instead of throwing an exception
  buffer_queue_full_action block
  # Set the chunk limit conservatively to avoid exceeding the GCL limit
  # of 10MiB per write request.
  buffer_chunk_limit 2M
  # Cap the combined memory usage of this buffer and the one below to
  # 2MiB/chunk * (6 + 2) chunks = 16 MiB
  buffer_queue_limit 6
  # Never wait more than 5 seconds before flushing logs in the non-error case.
  flush_interval 5s
  # Never wait longer than 30 seconds between retries.
  max_retry_wait 30
  # Disable the limit on the number of retries (retry forever).
  # disable_retry_limit
  # Use multiple threads for processing.
  num_threads 2
</match>
<% when "graylog"%>
# The gelf plugin assumes input in utf-8
<filter **>
  @type record_modifier
  char_encoding utf-8
</filter>

<match **>
  @type gelf
  @id out_graylog
  @log_level info
  include_tag_key true
  host "#{ENV['FLUENT_GRAYLOG_HOST']}"
  port "#{ENV['FLUENT_GRAYLOG_PORT']}"
  <buffer>
    flush_thread_count 8
    flush_interval 5s
    chunk_limit_size 8M
    queue_limit_length 512
    retry_max_interval 30<commit_msg>Made the protocol available to the graylog config<commit_after><% is_v1 = version.split("-").first.start_with?('v1') %>
<% target = (dockerfile.split("/").last.split("-").last) %>
<% is_alpine = (dockerfile.split("/").last.split("-").first == "alpine") %>

# AUTOMATICALLY GENERATED
# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb

@include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf"
@include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf"
@include kubernetes.conf
@include conf.d/*.conf

<% case target when "elasticsearch"%>
<match **>
   @type elasticsearch
   @id out_es
   @log_level info
   include_tag_key true
   host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
   port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
   path "#{ENV['FLUENT_ELASTICSEARCH_PATH']}"
   scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
   ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
   ssl_version "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERSION'] || 'TLSv1'}"
   user "#{ENV['FLUENT_ELASTICSEARCH_USER']}"
   password "#{ENV['<PASSWORD>']}"
   reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'false'}"
   reconnect_on_error "#{ENV['FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR'] || 'true'}"
   reload_on_failure "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE'] || 'true'}"
   log_es_400_reason "#{ENV['FLUENT_ELASTICSEARCH_LOG_ES_400_REASON'] || 'false'}"
   logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'logstash'}"
   logstash_format "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_FORMAT'] || 'true'}"
   index_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_INDEX_NAME'] || 'logstash'}"
   type_name "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_TYPE_NAME'] || 'fluentd'}"
<% if is_v1 %>
   <buffer>
     flush_thread_count "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
     flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
     chunk_limit_size "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
     queue_limit_length "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
     retry_max_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
     retry_forever true
   </buffer>
<% else %>
   buffer_chunk_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE'] || '2M'}"
   buffer_queue_limit "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_QUEUE_LIMIT_LENGTH'] || '32'}"
   flush_interval "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_INTERVAL'] || '5s'}"
   max_retry_wait "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_RETRY_MAX_INTERVAL'] || '30'}"
   disable_retry_limit
   num_threads "#{ENV['FLUENT_ELASTICSEARCH_BUFFER_FLUSH_THREAD_COUNT'] || '8'}"
<% end %>
</match>
<%  when "logentries"%>
<match **>
   @type logentries
   @id out_logentries
   use_json true
   tag_access_log stdout
   tag_error_log stderr
   config_path /etc/logentries/tokens.yaml
</match>
<%  when "loggly"%>
<match **>
   @type loggly
   @id out_loggly
   @log_level info
   loggly_url "https://logs-01.loggly.com/bulk/#{ENV['LOGGLY_TOKEN']}/tag/#{ENV['LOGGLY_TAGS'] || 'fluentd'}/bulk"
</match>
<%  when "cloudwatch"%>
<match **>
  @type cloudwatch_logs
  @id out_cloudwatch_logs
  log_group_name "#{ENV['LOG_GROUP_NAME']}"
  auto_create_stream true
  use_tag_as_stream true
<% if is_v1 %>
  json_handler yajl # To avoid UndefinedConversionError
  log_rejected_request "#{ENV['LOG_REJECTED_REQUEST']}" # Log rejected request for missing parts
<% end %>
</match>
<%  when "s3"%>
<match **>
  # docs: https://docs.fluentd.org/v0.12/articles/out_s3
  # note: this configuration relies on the nodes have an IAM instance profile with access to your S3 bucket
  @type s3
  @id out_s3
  @log_level info
  s3_bucket "#{ENV['S3_BUCKET_NAME']}"
  s3_region "#{ENV['S3_BUCKET_REGION']}"
<% if is_v1 %>
  s3_object_key_format %{path}%Y/%m/%d/cluster-log-%{index}.%{file_extension}
  <inject>
    time_key time
    tag_key tag
    localtime false
  </inject>
  <buffer>
    @type file
    path /var/log/fluentd-buffers/s3.buffer
    timekey 3600
    timekey_use_utc true
    chunk_limit_size 256m
  </buffer>
<% else %>
  s3_object_key_format %{path}%{time_slice}/cluster-log-%{index}.%{file_extension}
  time_slice_format %Y/%m/%d
  time_slice_wait 10m
  utc
  include_time_key true
  include_tag_key true
  buffer_chunk_limit 256m
  buffer_path /var/log/fluentd-buffers/s3.buffer
<% end %>
</match>
<%  when "gcs"%>
<match **>
  # docs: https://github.com/daichirata/fluent-plugin-gcs
  # this configuration relies on the nodes having permission to write on your gs bucket
  @type gcs
  @id out_gcs
  project "#{ENV['GCS_BUCKET_PROJECT']}"
  bucket "#{ENV['GCS_BUCKET_NAME']}"
  object_key_format %{path}%{time_slice}/%{hostname}/%{index}.%{file_extension}
  path logs/
  buffer_path /var/log/fluentd-buffers/gcs.buffer
  time_slice_format %Y/%m/%d
  time_slice_wait 10m
  utc
</match>
<% when "stackdriver"%>
<match **>
  @type google_cloud
  @id out_google_cloud

  # Set the buffer type to file to improve the reliability and reduce the memory consumption
  buffer_type file
  buffer_path /var/log/fluentd-buffers/stackdriver.buffer
  # Set queue_full action to block because we want to pause gracefully
  # in case of the off-the-limits load instead of throwing an exception
  buffer_queue_full_action block
  # Set the chunk limit conservatively to avoid exceeding the GCL limit
  # of 10MiB per write request.
  buffer_chunk_limit 2M
  # Cap the combined memory usage of this buffer and the one below to
  # 2MiB/chunk * (6 + 2) chunks = 16 MiB
  buffer_queue_limit 6
  # Never wait more than 5 seconds before flushing logs in the non-error case.
  flush_interval 5s
  # Never wait longer than 30 seconds between retries.
  max_retry_wait 30
  # Disable the limit on the number of retries (retry forever).
  # disable_retry_limit
  # Use multiple threads for processing.
  num_threads 2
</match>
<% when "graylog"%>
# The gelf plugin assumes input in utf-8
<filter **>
  @type record_modifier
  char_encoding utf-8
</filter>

<match **>
  @type gelf
  @id out_graylog
  @log_level info
  include_tag_key true
  host "#{ENV['FLUENT_GRAYLOG_HOST']}"
  port "#{ENV['FLUENT_GRAYLOG_PORT']}"
  protocol "#{ENV['FLUENT_GRAYLOG_PROTOCOL'] || 'udp'}"
  <buffer>
    flush_thread_count 8
    flush_interval 5s
    chunk_limit_size 8M
    queue_limit_length 512
    retry_max_interval 30
{'fraction_non_alphanumeric': 0.13687943262411348, 'fraction_numerical': 0.013238770685579196, 'mean_word_length': 5.15470417070805, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 91, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '100930', 'n_tokens_mistral': 4950, 'n_tokens_neox': 4803, 'n_words': 947}