<commit_before># Pick up all the auto-generated input config files, one for each file
# specified in the FILES_TO_COLLECT environment variable.
@include files/*

<system>
  log_level warn
</system>

# All the auto-generated files should use the tag "file.<filename>".
<source>
  @type systemd
  path /var/log/journal
  <storage>
    @type local
    persistent true
    path /var/log/opensearch_journal.pos
  </storage>
  tag systemd
  read_from_head true
</source>

<label @FLUENT_LOG>
  <match fluent.*>
    @type null
  </match>
</label>

# prometheus monitoring config

@include /fluentd/etc/prometheus.conf

<filter **>
  @type systemd_entry
#  field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
#  field_map_strict false
  fields_lowercase true
  fields_strip_underscores true
</filter>

<match **>
  @type copy
  <store>
    @type opensearch
{{- if eq .Values.global.clusterType  "scaleout" }}
    host {{.Values.endpoint_host_internal}}
    port {{.Values.http_port}}
    user {{.Values.user}}
{{- else }}
    hosts {{.Values.opensearch.http.endpoint}}.{{.Values.global.region}}.{{.Values.global.tld}}:443
    scheme https
    ssl_verify false
    ssl_version TLSv1_2
{{- end }}
    user {{.Values.user}}
    password {{.<PASSWORD>}}
    index_name systemd
    type_name _doc
    logstash_prefix systemd
    logstash_format true
    template_name systemd
    template_file /fluentd/etc/systemd.json
    template_overwrite true
    time_as_integer false
    @log_level info
    slow_flush_log_threshold 50.0
    request_timeout 60s
    include_tag_key true
    resurrect_after 120
    reconnect_on_error true
    reload_connections false
    reload_on_failure true
    <buffer>
      flush_at_shutdown true
      flush_thread_interval 5
      overflow_action block
      retry_forever true
      retry_wait 2s
      flush_thread_count 4
      flush_interval 3s
    </buffer>
  </store>
  <store>
    @type prometheus
    <metric>
      name fluentd_output_status_num_records_total
      type counter
      desc The total number of outgoing records
      <labels>
        nodename "#{ENV['NODENAME']}"
      </labels>
    </metric>
  </store>
</match>
<commit_msg>[opensearch] fix fluent-systemd for opensearch 2.x<commit_after># Pick up all the auto-generated input config files, one for each file
# specified in the FILES_TO_COLLECT environment variable.
@include files/*

<system>
  log_level warn
</system>

# All the auto-generated files should use the tag "file.<filename>".
<source>
  @type systemd
  path /var/log/journal
  <storage>
    @type local
    persistent true
    path /var/log/opensearch_journal.pos
  </storage>
  tag systemd
  read_from_head true
</source>

<label @FLUENT_LOG>
  <match fluent.*>
    @type null
  </match>
</label>

# prometheus monitoring config

@include /fluentd/etc/prometheus.conf

<filter **>
  @type systemd_entry
#  field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
#  field_map_strict false
  fields_lowercase true
  fields_strip_underscores true
</filter>

<match **>
  @type copy
  <store>
    @type opensearch
{{- if eq .Values.global.clusterType  "scaleout" }}
    host {{.Values.endpoint_host_internal}}
    port {{.Values.http_port}}
    user {{.Values.user}}
{{- else }}
    hosts {{.Values.opensearch.http.endpoint}}.{{.Values.global.region}}.{{.Values.global.tld}}:443
    scheme https
    ssl_verify false
    ssl_version TLSv1_2
{{- end }}
    user {{.Values.user}}
    password {{.<PASSWORD>}}
    index_name systemd
    type_name _doc
    logstash_prefix systemd
    logstash_format true
    template_name systemd
    template_file /fluentd/etc/systemd.json
    template_overwrite true
    time_as_integer false
    @log_level info
    slow_flush_log_threshold 50.0
    request_timeout 60s
    include_tag_key true
    resurrect_after 120
    reconnect_on_error true
    reload_connections false
    reload_on_failure true
    suppress_type_name true
    <buffer>
      flush_at_shutdown true
      flush_thread_interval 5
      overflow_action block
      retry_forever true
      retry_wait 2s
      flush_thread_count 4
      flush_interval 3s
    </buffer>
  </store>
  <store>
    @type prometheus
    <metric>
      name fluentd_output_status_num_records_total
      type counter
      desc The total number of outgoing records
      <labels>
        nodename "#{ENV['NODENAME']}"
      </labels>
    </metric>
  </store>
</match>

{'fraction_non_alphanumeric': 0.10282661918539951, 'fraction_numerical': 0.007789895392833296, 'mean_word_length': 3.223684210526316, 'pattern_counts': {'":': 8, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5831880', 'n_tokens_mistral': 1521, 'n_tokens_neox': 1471, 'n_words': 379}