<commit_before>########################## Metricbeat Configuration ###########################

#==========================  Modules configuration ============================
metricbeat.modules:

#------------------------------- System Module -------------------------------
- module: system
  enabled: ${SYSTEM_ENABLE}
  period: ${METRIC_PERIOD}
  metricsets:
    # CPU stats
    - cpu
    # Per CPU core stats
    - core
    # System Load stats
    - load
    # IO stats
    - diskio
    # Memory stats
    - memory
    # Processes summary
    # - process_summary
    # Per process stats
    # - process
  processes: ['.*']

  # if true, exports the CPU usage in ticks, together with the percentage values
  cpu_ticks: true

  # These options allow you to filter out all processes that are not
  # in the top N by CPU or memory, in order to reduce the number of documents created.
  # If both the `by_cpu` and `by_memory` options are used, the union of the two sets
  # is included.
  #process.include_top_n:
    #
    # Set to false to disable this feature and include all processes
    #enabled: true

    # How many processes to include from the top by CPU. The processes are sorted
    # by the `system.process.cpu.total.pct` field.
    #by_cpu: 0

    # How many processes to include from the top by memory. The processes are sorted
    # by the `system.process.memory.rss.bytes` field.
    #by_memory: 0

  # If false, cmdline of a process is not cached.
  #process.cmdline.cache.enabled: true

  # Enable collection of cgroup metrics from processes on Linux.
  #
  # NOTE: Pratically speaking, unless ur using custom cgroups, this means docker
  # This is disabled, as its only pratical from version 6 onwards
  #process.cgroups.enabled: ${DOCKER_ENABLE}

  # A list of regular expressions used to whitelist environment variables
  # reported with the process metricset's events. Defaults to empty.
  #process.env.whitelist: []

# Network specific monitoring
- module: system
  enabled: ${SYSTEM_ENABLE}
  period: ${METRIC_PERIOD}
  metricsets:
    # Network stats
    - network
    # Sockets and connection info (linux only)
    #- socket

  # Filterout excessive veth.* connectors, 
  # as it excessive show datausage for virtual drives
  # Which is common in AWS setup
  filters:
    - drop_event.when.regex.name: '^veth.*$'

  # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
  #socket.reverse_lookup.enabled: false
  #socket.reverse_lookup.success_ttl: 60s
  #socket.reverse_lookup.failure_ttl: 60s

# File system specific monitoring (reduced)
- module: system
  enabled: ${SYSTEM_ENABLE}
  period: ${FS_METRIC_PERIOD}
  metricsets:
    # Per filesystem stats
    - filesystem
    # File system summary stats
    - fsstat
  filters:
    - drop_event.when.regexp.mount_point: '^/(hostfs|sys|cgroup|proc|dev|etc|host|lib)($|/|;)'

#------------------------------- Docker Module -------------------------------
- module: docker
  metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
  hosts: ["unix:///hostfs/var/run/docker.sock"]
  enabled: ${DOCKER_ENABLE}
  period: ${METRIC_PERIOD}

  # To connect to Docker over TLS you must specify a client and CA certificate.
  #ssl:
    #certificate_authority: "/etc/pki/root/ca.pem"
    #certificate:           "/etc/pki/client/cert.pem"
    #key:                   "/etc/pki/client/cert.key"

#================================ General ======================================

# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
# If this options is not defined, the hostname is used.
#name:

# The tags of the shipper are included in their own field with each
# transaction published. Tags make it easy to group servers by different
# logical properties.
#tags: ["service-X", "web-tier"]

# Optional fields that you can specify to add additional information to the
# output. Fields can be scalar values, arrays, dictionaries, or any nested
# combination of these.
#fields:
#  env: staging

# If this option is set to true, the custom fields are stored as top-level
# fields in the output document instead of being grouped under a fields
# sub-dictionary. Default is false.
#fields_under_root: false

# Internal queue size for single events in processing pipeline
#queue_size: 1000

# The internal queue size for bulk events in the processing pipeline.
# Do not modify this value.
#bulk_queue_size: 0

# Sets the maximum number of CPUs that can be executing simultaneously. The
# default is the number of logical CPUs available in the system.
#max_procs:

#================================ Processors ===================================

# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
# processors that are applied one by one and the first one receives the initial
# event:
#
#   event -> filter1 -> event1 -> filter2 ->event2 ...
#
# The supported processors are drop_fields, drop_event, include_fields, and
# add_cloud_metadata.
#
# For example, you can use the following processors to keep the fields that
# contain CPU load percentages, but remove the fields that contain CPU ticks
# values:
#
#processors:
#- include_fields:
#    fields: ["cpu"]
#- drop_fields:
#    fields: ["cpu.user", "cpu.system"]
#
# The following example drops the events that have the HTTP response code 200:
#
#processors:
#- drop_event:
#    when:
#       equals:
#           http.code: 200
#
# The following example enriches each event with metadata from the cloud
# provider about the host machine. It works on EC2, GCE, DigitalOcean,
# Tencent Cloud, and Alibaba Cloud.
#
processors:
- add_cloud_metadata: ~
#
# The following example enriches each event with the machine's local time zone
# offset from UTC.
#
#processors:
#- add_locale:
#    format: offset
#
# The following example enriches each event with docker metadata, it matches
# given fields to an existing container id and adds info from that container:
#
# processors:
# - add_docker_metadata:
#    host: "unix:///hostfs/var/run/docker.sock"
#    match_source: true
#    match_source_index: 4
#    match_fields: ["system.process.cgroup.id"]
#  # To connect to Docker over TLS you must specify a client and CA certificate.
#  #ssl:
#  #  certificate_authority: "/etc/pki/root/ca.pem"
#  #  certificate:           "/etc/pki/client/cert.pem"
#  #  key:                   "/etc/pki/client/cert.key"


#================================ Outputs ======================================

# Configure what outputs to use when sending the data collected by the beat.
# Multiple outputs may be used.

#-------------------------- Elasticsearch output -------------------------------
output.elasticsearch:
  # Boolean flag to enable or disable the output module.
  #enabled: true

  # Array of hosts to connect to.
  # Scheme and port can be left out and will be set to the default (http and 9200)
  # In case you specify and additional path, the scheme is required: http://localhost:9200/path
  # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
  hosts: ["${ES_HOST}:${ES_PORT}"]

  # Set gzip compression level.
  #compression_level: 0

  # Optional protocol and basic auth credentials.
  protocol: "${ES_PROT}"
  username: "${ES_USER}"
  password: "${<PASSWORD>}"

  # Dictionary of HTTP parameters to pass within the url with index operations.
  #parameters:
    #param1: value1
    #param2: value2

  # Number of workers per Elasticsearch host.
  #worker: 1

  # Optional index name. The default is "metricbeat" plus date
  # and generates [metricbeat-]YYYY.MM.DD keys.
  #index: "metricbeat-%{[beat.version]}-%{+yyyy.MM.dd}"
  index: "${ES_INDEX}"

  # Optional ingest node pipeline. By default no pipeline will be used.
  #pipeline: ""

  # Optional HTTP Path
  #path: "/elasticsearch"

  # Custom HTTP headers to add to each request
  #headers:
  #  X-My-Header: Contents of the header

  # Proxy server url
  #proxy_url: http://proxy:3128

  # The number of times a particular Elasticsearch index operation is attempted. If
  # the indexing operation doesn't succeed after this many retries, the events are
  # dropped. The default is 3.
  #max_retries: 3

  # The maximum number of events to bulk in a single Elasticsearch bulk API index request.
  # The default is 50.
  #bulk_max_size: 50

  # Configure http request timeout before failing an request to Elasticsearch.
  #timeout: 90

  # The number of seconds to wait for new events between two bulk API index requests.
  # If `bulk_max_size` is reached before this interval expires, addition bulk index
  # requests are made.
  #flush_interval: 1s

  # Use SSL settings for HTTPS. Default is true.
  #ssl.enabled: true

  # Configure SSL verification mode. If `none` is configured, all server hosts
  # and certificates will be accepted. In this mode, SSL based connections are
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
  # `full`.
  #ssl.verification_mode: full

  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
  # 1.2 are enabled.
  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]

  # SSL configuration. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

  # Optional passphrase for decrypting the Certificate Key.
  #ssl.key_passphrase: ''

  # Configure cipher suites to be used for SSL connections
  #ssl.cipher_suites: []

  # Configure curve types for ECDHE based cipher suites
  #ssl.curve_types: []

#----------------------------- Console output ---------------------------------
#output.console:
  # Boolean flag to enable or disable the output module.
  #enabled: true

  # Pretty print json event
  #pretty: false

#================================ Logging ======================================
# There are three options for the log output: syslog, file, stderr.
# Under Windows systems, the log files are per default sent to the file output,
# under all other system per default to syslog.

# Sets log level. The default log level is info.
# Available log levels are: critical, error, warning, info, debug
logging.level: ${LOGGING_LEVEL}

# Enable debug output for selected components. To enable all selectors use ["*"]
# Other available selectors are "beat", "publish", "service"
# Multiple selectors can be chained.
#logging.selectors: ["*"]

# Send all logging output to syslog. The default is false.
#logging.to_syslog: true

# If enabled, metricbeat periodically logs its internal metrics that have changed
# in the last period. For each metric that changed, the delta from the value at
# the beginning of the period is logged. Also, the total values for
# all non-zero internal metrics are logged on shutdown. The default is true.
#logging.metrics.enabled: true

# The period after which to log the internal metrics. The default is 30s.
#logging.metrics.period: 30s

# # Logging to rotating files files. Set logging.to_files to false to disable logging to
# # files.
# logging.to_files: true
# logging.files:
#   # Configure the path where the logs are written. The default is the logs directory
#   # under the home path (the binary location).
#   path: /var/log/metricbeat

#   # The name of the files where the logs are written to.
#   name: metricbeat

#   # Configure log file size limit. If limit is reached, log file will be
#   # automatically rotated
#   rotateeverybytes: 10485760 # = 10MB

#   # Number of rotated log files to keep. Oldest files will be deleted first.
#   keepfiles: 10<commit_msg>Tweak filesystem filter, to include rancher<commit_after>########################## Metricbeat Configuration ###########################

#==========================  Modules configuration ============================
metricbeat.modules:

#------------------------------- System Module -------------------------------
- module: system
  enabled: ${SYSTEM_ENABLE}
  period: ${METRIC_PERIOD}
  metricsets:
    # CPU stats
    - cpu
    # Per CPU core stats
    - core
    # System Load stats
    - load
    # IO stats
    - diskio
    # Memory stats
    - memory
    # Processes summary
    # - process_summary
    # Per process stats
    # - process
  processes: ['.*']

  # if true, exports the CPU usage in ticks, together with the percentage values
  cpu_ticks: true

  # These options allow you to filter out all processes that are not
  # in the top N by CPU or memory, in order to reduce the number of documents created.
  # If both the `by_cpu` and `by_memory` options are used, the union of the two sets
  # is included.
  #process.include_top_n:
    #
    # Set to false to disable this feature and include all processes
    #enabled: true

    # How many processes to include from the top by CPU. The processes are sorted
    # by the `system.process.cpu.total.pct` field.
    #by_cpu: 0

    # How many processes to include from the top by memory. The processes are sorted
    # by the `system.process.memory.rss.bytes` field.
    #by_memory: 0

  # If false, cmdline of a process is not cached.
  #process.cmdline.cache.enabled: true

  # Enable collection of cgroup metrics from processes on Linux.
  #
  # NOTE: Pratically speaking, unless ur using custom cgroups, this means docker
  # This is disabled, as its only pratical from version 6 onwards
  #process.cgroups.enabled: ${DOCKER_ENABLE}

  # A list of regular expressions used to whitelist environment variables
  # reported with the process metricset's events. Defaults to empty.
  #process.env.whitelist: []

# Network specific monitoring
- module: system
  enabled: ${SYSTEM_ENABLE}
  period: ${METRIC_PERIOD}
  metricsets:
    # Network stats
    - network
    # Sockets and connection info (linux only)
    #- socket

  # Filterout excessive veth.* connectors, 
  # as it excessive show datausage for virtual drives
  # Which is common in AWS setup
  filters:
    - drop_event.when.regex.name: '^veth.*$'

  # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
  #socket.reverse_lookup.enabled: false
  #socket.reverse_lookup.success_ttl: 60s
  #socket.reverse_lookup.failure_ttl: 60s

# File system specific monitoring (reduced)
- module: system
  enabled: ${SYSTEM_ENABLE}
  period: ${FS_METRIC_PERIOD}
  metricsets:
    # Per filesystem stats
    - filesystem
    # File system summary stats
    - fsstat
  filters:
    - drop_event.when.regexp.mount_point: '^/(\.r|hostfs|sys|cgroup|proc|dev|etc|host|lib)($|/|;)'

#------------------------------- Docker Module -------------------------------
- module: docker
  metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
  hosts: ["unix:///hostfs/var/run/docker.sock"]
  enabled: ${DOCKER_ENABLE}
  period: ${METRIC_PERIOD}

  # To connect to Docker over TLS you must specify a client and CA certificate.
  #ssl:
    #certificate_authority: "/etc/pki/root/ca.pem"
    #certificate:           "/etc/pki/client/cert.pem"
    #key:                   "/etc/pki/client/cert.key"

#================================ General ======================================

# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
# If this options is not defined, the hostname is used.
#name:

# The tags of the shipper are included in their own field with each
# transaction published. Tags make it easy to group servers by different
# logical properties.
#tags: ["service-X", "web-tier"]

# Optional fields that you can specify to add additional information to the
# output. Fields can be scalar values, arrays, dictionaries, or any nested
# combination of these.
#fields:
#  env: staging

# If this option is set to true, the custom fields are stored as top-level
# fields in the output document instead of being grouped under a fields
# sub-dictionary. Default is false.
#fields_under_root: false

# Internal queue size for single events in processing pipeline
#queue_size: 1000

# The internal queue size for bulk events in the processing pipeline.
# Do not modify this value.
#bulk_queue_size: 0

# Sets the maximum number of CPUs that can be executing simultaneously. The
# default is the number of logical CPUs available in the system.
#max_procs:

#================================ Processors ===================================

# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
# processors that are applied one by one and the first one receives the initial
# event:
#
#   event -> filter1 -> event1 -> filter2 ->event2 ...
#
# The supported processors are drop_fields, drop_event, include_fields, and
# add_cloud_metadata.
#
# For example, you can use the following processors to keep the fields that
# contain CPU load percentages, but remove the fields that contain CPU ticks
# values:
#
#processors:
#- include_fields:
#    fields: ["cpu"]
#- drop_fields:
#    fields: ["cpu.user", "cpu.system"]
#
# The following example drops the events that have the HTTP response code 200:
#
#processors:
#- drop_event:
#    when:
#       equals:
#           http.code: 200
#
# The following example enriches each event with metadata from the cloud
# provider about the host machine. It works on EC2, GCE, DigitalOcean,
# Tencent Cloud, and Alibaba Cloud.
#
processors:
- add_cloud_metadata: ~
#
# The following example enriches each event with the machine's local time zone
# offset from UTC.
#
#processors:
#- add_locale:
#    format: offset
#
# The following example enriches each event with docker metadata, it matches
# given fields to an existing container id and adds info from that container:
#
# processors:
# - add_docker_metadata:
#    host: "unix:///hostfs/var/run/docker.sock"
#    match_source: true
#    match_source_index: 4
#    match_fields: ["system.process.cgroup.id"]
#  # To connect to Docker over TLS you must specify a client and CA certificate.
#  #ssl:
#  #  certificate_authority: "/etc/pki/root/ca.pem"
#  #  certificate:           "/etc/pki/client/cert.pem"
#  #  key:                   "/etc/pki/client/cert.key"


#================================ Outputs ======================================

# Configure what outputs to use when sending the data collected by the beat.
# Multiple outputs may be used.

#-------------------------- Elasticsearch output -------------------------------
output.elasticsearch:
  # Boolean flag to enable or disable the output module.
  #enabled: true

  # Array of hosts to connect to.
  # Scheme and port can be left out and will be set to the default (http and 9200)
  # In case you specify and additional path, the scheme is required: http://localhost:9200/path
  # IPv6 addresses should always be defined as: https://[2001:fc00:e968:6179::de52:7100]:9200
  hosts: ["${ES_HOST}:${ES_PORT}"]

  # Set gzip compression level.
  #compression_level: 0

  # Optional protocol and basic auth credentials.
  protocol: "${ES_PROT}"
  username: "${ES_USER}"
  password: "${<PASSWORD>}"

  # Dictionary of HTTP parameters to pass within the url with index operations.
  #parameters:
    #param1: value1
    #param2: value2

  # Number of workers per Elasticsearch host.
  #worker: 1

  # Optional index name. The default is "metricbeat" plus date
  # and generates [metricbeat-]YYYY.MM.DD keys.
  #index: "metricbeat-%{[beat.version]}-%{+yyyy.MM.dd}"
  index: "${ES_INDEX}"

  # Optional ingest node pipeline. By default no pipeline will be used.
  #pipeline: ""

  # Optional HTTP Path
  #path: "/elasticsearch"

  # Custom HTTP headers to add to each request
  #headers:
  #  X-My-Header: Contents of the header

  # Proxy server url
  #proxy_url: http://proxy:3128

  # The number of times a particular Elasticsearch index operation is attempted. If
  # the indexing operation doesn't succeed after this many retries, the events are
  # dropped. The default is 3.
  #max_retries: 3

  # The maximum number of events to bulk in a single Elasticsearch bulk API index request.
  # The default is 50.
  #bulk_max_size: 50

  # Configure http request timeout before failing an request to Elasticsearch.
  #timeout: 90

  # The number of seconds to wait for new events between two bulk API index requests.
  # If `bulk_max_size` is reached before this interval expires, addition bulk index
  # requests are made.
  #flush_interval: 1s

  # Use SSL settings for HTTPS. Default is true.
  #ssl.enabled: true

  # Configure SSL verification mode. If `none` is configured, all server hosts
  # and certificates will be accepted. In this mode, SSL based connections are
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
  # `full`.
  #ssl.verification_mode: full

  # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
  # 1.2 are enabled.
  #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]

  # SSL configuration. By default is off.
  # List of root certificates for HTTPS server verifications
  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication
  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key
  #ssl.key: "/etc/pki/client/cert.key"

  # Optional passphrase for decrypting the Certificate Key.
  #ssl.key_passphrase: ''

  # Configure cipher suites to be used for SSL connections
  #ssl.cipher_suites: []

  # Configure curve types for ECDHE based cipher suites
  #ssl.curve_types: []

#----------------------------- Console output ---------------------------------
#output.console:
  # Boolean flag to enable or disable the output module.
  #enabled: true

  # Pretty print json event
  #pretty: false

#================================ Logging ======================================
# There are three options for the log output: syslog, file, stderr.
# Under Windows systems, the log files are per default sent to the file output,
# under all other system per default to syslog.

# Sets log level. The default log level is info.
# Available log levels are: critical, error, warning, info, debug
logging.level: ${LOGGING_LEVEL}

# Enable debug output for selected components. To enable all selectors use ["*"]
# Other available selectors are "beat", "publish", "service"
# Multiple selectors can be chained.
#logging.selectors: ["*"]

# Send all logging output to syslog. The default is false.
#logging.to_syslog: true

# If enabled, metricbeat periodically logs its internal metrics that have changed
# in the last period. For each metric that changed, the delta from the value at
# the beginning of the period is logged. Also, the total values for
# all non-zero internal metrics are logged on shutdown. The default is true.
#logging.metrics.enabled: true

# The period after which to log the internal metrics. The default is 30s.
#logging.metrics.period: 30s

# # Logging to rotating files files. Set logging.to_files to false to disable logging to
# # files.
# logging.to_files: true
# logging.files:
#   # Configure the path where the logs are written. The default is the logs directory
#   # under the home path (the binary location).
#   path: /var/log/metricbeat

#   # The name of the files where the logs are written to.
#   name: metricbeat

#   # Configure log file size limit. If limit is reached, log file will be
#   # automatically rotated
#   rotateeverybytes: 10485760 # = 10MB

#   # Number of rotated log files to keep. Oldest files will be deleted first.
#   keepfiles: 10
{'fraction_non_alphanumeric': 0.1326877122374544, 'fraction_numerical': 0.007923531631241354, 'mean_word_length': 3.3833149577361263, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '659896', 'n_tokens_mistral': 6946, 'n_tokens_neox': 6598, 'n_words': 2786}