<commit_before>input {
  http_poller {
    urls => {
      getToken => {
        url => "{{.Values.http_poller.url.token}}"
        method => POST
        user => "{{.Values.http_poller.user}}"
        password => "{{.Values.http_poller.password}}"
     }
  }
  target => "token_response"
  request_timeout => 60
  schedule => { cron => "0 */1 * * *"}
  codec => "json"
  }
}
filter {

 ruby {
    init => "require 'time'"
    code => '
             upper = Time.now
             upper = upper - upper.sec - 60 * upper.min
             lower = upper - 3600
             lower = lower.strftime "%Y-%m-%dT%H.%M.%S"
             upper = upper.strftime "%Y-%m-%dT%H.%M.%S"
             event.set("[timerange][lower]", lower)
             event.set("[timerange][upper]", upper)
            '
  }

  http {
    body_format => "json"
    follow_redirects => true
    url => "{{.Values.http_poller.url.api}}?$filter=Time%20gt%20'%{[timerange][lower]}'%20and%20Time%20le%20'%{[timerange][upper]}'"
    verb => "GET"
    headers => { "Authorization" => "Bearer %{[token_response][access_token]}" }
    request_timeout => 60
    socket_timeout => 30
  }

  split{
    field => "[body][value]"
    target => "event"
  }

  kv {
     source => "[event][Message]"
     target => "[event][details]"
     field_split_pattern => "," 
     }

  ruby {
    code => '
      event.get("[event][details]").to_hash.keys.each { |k|
      if k.start_with?('202')
        event.remove("[event][details]" + "[" + k + "]")
      end
      }<commit_msg>[ELK] Remove timestamp from audit event details<commit_after>input {
  http_poller {
    urls => {
      getToken => {
        url => "{{.Values.http_poller.url.token}}"
        method => POST
        user => "{{.Values.http_poller.user}}"
        password => "{{.Values.http_poller.password}}"
     }
  }
  target => "token_response"
  request_timeout => 60
  schedule => { cron => "0 */1 * * *"}
  codec => "json"
  }
}
filter {

 ruby {
    init => "require 'time'"
    code => '
             upper = Time.now
             upper = upper - upper.sec - 60 * upper.min
             lower = upper - 3600
             lower = lower.strftime "%Y-%m-%dT%H.%M.%S"
             upper = upper.strftime "%Y-%m-%dT%H.%M.%S"
             event.set("[timerange][lower]", lower)
             event.set("[timerange][upper]", upper)
            '
  }

  http {
    body_format => "json"
    follow_redirects => true
    url => "{{.Values.http_poller.url.api}}?$filter=Time%20gt%20'%{[timerange][lower]}'%20and%20Time%20le%20'%{[timerange][upper]}'"
    verb => "GET"
    headers => { "Authorization" => "Bearer %{[token_response][access_token]}" }
    request_timeout => 60
    socket_timeout => 30
  }

  split{
    field => "[body][value]"
    target => "event"
  }

  kv {
     source => "[event][Message]"
     target => "[event][details]"
     field_split_pattern => "," 
     }

  ruby {
    code => '
      event.get("[event][details]").to_hash.keys.each { |k|
      if k.start_with?("202")
        event.remove("[event][details]" + "[" + k + "]")
      end
      }
{'fraction_non_alphanumeric': 0.19402985074626866, 'fraction_numerical': 0.01881894873458793, 'mean_word_length': 2.201453790238837, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1114956', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1045, 'n_words': 176}