<commit_before>filter {

    <% if 'DEBUG' == p('logstash.metadata_level') %>
        mutate {
            add_field => [ "@parser[job]", "<%= name %>/<%= index %>" ]
        }

        ruby {
            code => "event['@parser[timestamp]'] = Time.now.iso8601(3)"
        }
    <% end %>

    #
    # Default type to _logstash_input 
    #

    alter {
        coalesce => [
            "type", "%{_logstash_input}", "%{_type}"
        ]
    }

    #
    # rewrite our defined globals
    #

    if [type] == 'redis' or [type] == 'redis-input' {
        mutate {
            remove_field => [ 'type' ]
        }
    }

    if [type] != '' {
        mutate {
            rename => [ "type", "@type" ]
        }
    }

    if [message] != '' {
        mutate {
            rename => [ "message", "@message" ]
        }
    } else if [message] == '' and [@message] !~ /^.+$/ {
        drop { }
    }

    #
    # ignore particularly useless lines
    #

    if [@message] =~ /^\s*$/ or [@message] =~ /^#.*$/ {
        drop { }
    }

    #
    # trim excessively long messages
    #

    ruby {
        code => "(event['@message'] = event['@message'][0,<%= p("logstash_parser.message_max_size") %>] and (event['tags'] ||= []) << '_groktrimmed') if event['@message'] and event['@message'].length > <%= p("logstash_parser.message_max_size") %>"
    }

    #
    # trim excess whitespace
    #

    mutate {
        strip => [ "@message" ]
    }

    #
    # Additional filter types from deployment manifest
    #<commit_msg>Avoid indices literally named %{@type}<commit_after>filter {

    <% if 'DEBUG' == p('logstash.metadata_level') %>
        mutate {
            add_field => [ "@parser[job]", "<%= name %>/<%= index %>" ]
        }

        ruby {
            code => "event['@parser[timestamp]'] = Time.now.iso8601(3)"
        }
    <% end %>

    #
    # Default type to _logstash_input 
    #

    alter {
        coalesce => [
            "type", "%{_logstash_input}", "%{_type}"
        ]
    }

    #
    # rewrite our defined globals
    #

    if [type] == 'redis' or [type] == 'redis-input' {
        mutate {
            remove_field => [ 'type' ]
        }
    }

    if [type] != '' {
        mutate {
            rename => [ "type", "@type" ]
        }
    }

    if [message] != '' {
        mutate {
            rename => [ "message", "@message" ]
        }
    } else if [message] == '' and [@message] !~ /^.+$/ {
        drop { }
    }

    #
    # avoid bad interpolations, like `%{type}` when its missing
    #

    if [@type] == "" {
        mutate {
            add_field => [ "@type", "unknown" ]
        }
    }

    #
    # ignore particularly useless lines
    #

    if [@message] =~ /^\s*$/ or [@message] =~ /^#.*$/ {
        drop { }
    }

    #
    # trim excessively long messages
    #

    ruby {
        code => "(event['@message'] = event['@message'][0,<%= p("logstash_parser.message_max_size") %>] and (event['tags'] ||= []) << '_groktrimmed') if event['@message'] and event['@message'].length > <%= p("logstash_parser.message_max_size") %>"
    }

    #
    # trim excess whitespace
    #

    mutate {
        strip => [ "@message" ]
    }

    #
    # Additional filter types from deployment manifest
    #
{'fraction_non_alphanumeric': 0.18971358176778566, 'fraction_numerical': 0.0036957191253464737, 'mean_word_length': 1.6865177832919769, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1884407', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1059, 'n_words': 204}