<commit_before>{% from "fluentd/record_tagging.jinja" import record_tagging with context %}
{% from "fluentd/auth_log.jinja" import auth_log_source, auth_log_filter with context %}

fluentd:
  overrides:
    pkgs:
      - ruby2.3
      - ruby2.3-dev
      - build-essential
  plugins:
    - fluent-plugin-secure-forward
  configs:
    - name: edx
      settings:
        - directive: source
          attrs:
            - '@id': edx_nginx_access_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.nginx.access
            - path: /edx/var/log/nginx/access.log
            - pos_file: /edx/var/log/nginx/access.log.pos
            - nested_directives:
                - directive: parse
                  attrs:
                    - '@type': ltsv
                    - keep_time_key: 'true'
                    - label_delimiter: '='
                    - delimiter_pattern: '/\s+(?=(?:[^"]*"[^"]*")*[^"]*$)/'
                    - time_key: time
                    - types: time:time
        - directive: source
          attrs:
            - '@id': edx_cms_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.cms
            - path: /edx/var/log/cms/edx.log
            - pos_file: /edx/var/log/cms/edx.log.pos
            - format: multiline
            - format_firstline: '/^\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}/'
            - format1: '/^(?<time>\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}) (?<hostname>[^ ]+?) \[service_variant=(?<service_variant>\w+?)\]\[(?<namespace>[a-zA-Z._-]+?)\]\[env:(?<logging_env>[a-zA-Z-_.]+)\]\ (?<log_level>\w+) \[[^ ]+\s+\d+\] \[(?<filename>[a-zA-Z0-9-_.]+):(?<line_number>\d+)\] - (?<message>.*)$/'
            - time_format: '%b %d %H:%M:%S'
            - multiline_flush_interval: '5s'
        - directive: source
          attrs:
            - '@id': edx_lms_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.lms
            - path: /edx/var/log/lms/edx.log
            - pos_file: /edx/var/log/lms/edx.log.pos
            - format: multiline
            - format_firstline: '/^\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}/'
            - format1: '/^(?<time>\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}) (?<hostname>[^ ]+?) \[service_variant=(?<service_variant>\w+?)\]\[(?<namespace>[a-zA-Z._-]+?)\]\[env:(?<logging_env>[a-zA-Z-_.]+)\]\ (?<log_level>\w+) \[[^ ]+\s+\d+\] \[(?<filename>[a-zA-Z0-9-_.]+):(?<line_number>\d+)\] - (?<message>.*)$/'
            - time_format: '%b %d %H:%M:%S'
            - multiline_flush_interval: '5s'
        - directive: source
          attrs:
            - '@id': edx_xqueue_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.xqueue
            - path: /edx/var/log/xqueue/edx.log
            - pos_file: /edx/var/log/xqueue/edx.log.pos
            - format: multiline
            - format_firstline: '/^\w{3}\s+\d{1,2}\s+\d{1,2}:\d{2}:\d{2}/'
            - format1: '/^(?<time>\w{3}\s+\d{1,2}\s+\d{1,2}:\d{2}:\d{2}) (?<hostname>[^ ]+?) \[service_variant=(?<service_variant>\w+?)\]\[(?<namespace>[a-zA-Z._-]+?)\]\[env:(?<logging_env>[a-zA-Z-_.]+)\]\ (?<log_level>\w+) \[[^ ]+\s+\d+\] \[(?<filename>[a-zA-Z0-9-_.]+):(?<line_number>\d+)\] - (?<message>.*)$/'
            - time_format: '%b %d %H:%M:%S'
            - multiline_flush_interval: '5s'
        {% if salt.file.directory_exists('/edx/var/log/gr') %}
        - directive: source<commit_msg>Update mitx.sls<commit_after>{% from "fluentd/record_tagging.jinja" import record_tagging with context %}
{% from "fluentd/auth_log.jinja" import auth_log_source, auth_log_filter with context %}

fluentd:
  overrides:
    pkgs:
      - ruby2.3
      - ruby2.3-dev
      - build-essential
  plugins:
    - fluent-plugin-secure-forward
  configs:
    - name: edx
      settings:
        - directive: source
          attrs:
            - '@id': edx_nginx_access_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.nginx.access
            - path: /edx/var/log/nginx/access.log
            - pos_file: /edx/var/log/nginx/access.log.pos
            - nested_directives:
                - directive: parse
                  attrs:
                    - '@type': ltsv
                    - keep_time_key: 'true'
                    - label_delimiter: '='
                    - delimiter_pattern: '/\s+(?=(?:[^"]*"[^"]*")*[^"]*$)/'
                    - time_key: time
                    - types: time:time
        - directive: source
          attrs:
            - '@id': edx_cms_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.cms
            - path: /edx/var/log/cms/edx.log
            - pos_file: /edx/var/log/cms/edx.log.pos
            - format: multiline
            - format_firstline: '/^\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}/'
            - format1: '/^(?<time>\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}) (?<hostname>[^ ]+?) \[service_variant=(?<service_variant>\w+?)\]\[(?<namespace>[a-zA-Z._-]+?)\]\[env:(?<logging_env>[a-zA-Z-_.]+)\]\ (?<log_level>\w+) \[[^ ]+\s+\d+\] \[(?<filename>[a-zA-Z0-9-_.]+):(?<line_number>\d+)\] - (?<message>.*)$/'
            - time_format: '%b %d %H:%M:%S'
            - multiline_flush_interval: '5s'
        - directive: source
          attrs:
            - '@id': edx_lms_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.lms
            - path: /edx/var/log/lms/edx.log
            - pos_file: /edx/var/log/lms/edx.log.pos
            - format: multiline
            - format_firstline: '/^\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}/'
            - format1: '/^(?<time>\w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2}) (?<hostname>[^ ]+?) \[service_variant=(?<service_variant>\w+?)\]\[(?<namespace>[a-zA-Z._-]+?)\]\[env:(?<logging_env>[a-zA-Z-_.]+)\]\ (?<log_level>\w+) \[[^ ]+\s+\d+\] \[(?<filename>[a-zA-Z0-9-_.]+):(?<line_number>\d+)\] - (?<message>.*)$/'
            - time_format: '%b %d %H:%M:%S'
            - multiline_flush_interval: '5s'
        - directive: source
          attrs:
            - '@id': edx_xqueue_log
            - '@type': tail
            - enable_watch_timer: 'false'
            - tag: edx.xqueue
            - path: /edx/var/log/xqueue/edx.log
            - pos_file: /edx/var/log/xqueue/edx.log.pos
            - format: multiline
            - format_firstline: '/^\w{3}\s+\d{1,2}\s+\d{1,2}:\d{2}:\d{2}/'
            - format1: '/^(?<time>\w{3}\s+\d{1,2}\s+\d{1,2}:\d{2}:\d{2}) (?<hostname>[^ ]+?) \[service_variant=(?<service_variant>\w+?)\]\[(?<namespace>[a-zA-Z._-]+?)\]\[env:(?<logging_env>[a-zA-Z-_.]+)\]\ (?<log_level>\w+) \[[^ ]+\s+\d+\] \[(?<filename>[a-zA-Z0-9-_.]+):(?<line_number>\d+)\] - (?<message>.*)$/'
            - time_format: '%b %d %H:%M:%S'
            - multiline_flush_interval: '5s'
        {% if salt.file.file_exists('/edx/var/log/gr/gitreload.log') %}
        - directive: source
{'fraction_non_alphanumeric': 0.23883383599023408, 'fraction_numerical': 0.015510555794915984, 'mean_word_length': 1.723504106374658, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479345', 'n_tokens_mistral': 2825, 'n_tokens_neox': 2654, 'n_words': 326}