<commit_before>input {

# consume Keystone notifications
rabbitmq {
    # only for keystone
    host => "{{.Values.hermes.rabbitmq.keystone.host}}"
    password => "{{.Values.hermes.rabbitmq.keystone.password}}"
    # the remaining parameters are equal
    user => "{{.Values.hermes.rabbitmq.user}}"
    port => {{.Values.hermes.rabbitmq.port}}
    queue => "{{.Values.hermes.rabbitmq.queue_name}}"
    subscription_retry_interval_seconds => 60
    id => "logstash_hermes_keystone"
    automatic_recovery => false
  }

# consume Nova notifications
# rabbitmq {
#     host => "{{.Values.hermes_nova_rabbitmq_host}}"
#     user => "{{.Values.hermes_nova_rabbitmq_user}}"
#     password => "{{.Values.her<PASSWORD>_<PASSWORD>_<PASSWORD>_password}}"
#     port => {{.Values.hermes_nova_rabbitmq_port}}
#     queue => "{{.Values.hermes_nova_rabbitmq_queue_name}}"
#     subscription_retry_interval_seconds => 60
#     id => "logstash_hermes_nova"
#     automatic_recovery => false
#   }
# }
}


filter {
  # Strip oslo envelope
  mutate {
    remove_field => [ "publisher_id", "event_type", "message_id", "priority", "timestamp" ] 
  }
  ruby {
    code => "
      event.get('payload').each {|k, v|
        event.set(k, v)
      }
      event.remove('payload')
    "
  }

  # KEYSTONE TRANSFORMATIONS
  # Keystone specific transformations to compensate for scope missing from initiator element
  # When scope is missing from initiator, get it from action-specific parameters
  if ![initiator][project_id] and ![initiator][domain_id] {
    if [project] {
      mutate { rename => { "%{[project]}" => "%{[initiator][project_id]}" } }
    } else if [domain] {
      mutate { add_field => { "[initiator][domain_id]" => "%{[initiator][domain_id]}" } }
    }
  }

  # rename initiator user_id into the 'id' field for consistency
  if [initiator][user_id] {
    mutate {
      replace => { "[initiator][id]" => "%{[initiator][user_id]}" } 
      remove_field => ["[initiator][user_id]"]
    }
  }

  # add target projects/domains as attachment
  if ![target][attachments] {
    if [project] {
      ruby {
        code => "
          attachments = event.get('[target][attachments]')
          if attachments.nil?
            attachments = []
          end
          attachments << { 'name' => 'project_id', 'contentType' => '/data/security/project', 'content' => event.get('project') }
          event.set('[target][attachments]', attachments)
        "
    } else if [domain] {
      ruby {
        code => "
          attachments = event.get('[target][attachments]')
          if attachments.nil?
            attachments = []
          end
          attachments << { 'name' => 'domain_id', 'contentType' => '/data/security/domain', 'content' => event.get('domain') }
          event.set('[target][attachments]', attachments)
        "
      }
    }
  }

  if [target][typeURI] == "service/security/account/user" and [user] {
     mutate {
       replace => { "[target][id]" => "%{[user]}" }
       remove_field => ["[user]"]
     }
  }

  # Calculate the variable index name part from payload (@metadata will not be part of the event)

  # primary index
  if [initiator][project_id] {
    mutate { add_field => { "[@metadata][index]" => "%{[initiator][project_id]}" } }
  } else if [initiator][domain_id] {
    mutate { add_field => { "[@metadata][index]" => "%{[initiator][domain_id]}" } }
  }

  # secondary index (keystone only)
  if [project] {
    mutate { add_field => { "[@metadata][index2]" => "%{[project]}" } }
  } else if [domain] {
    mutate { add_field => { "[@metadata][index2]" => "%{[domain]}" } }
  }

  # remove keystone specific fields after they have been mapped to standard attachments 
  mutate {
    remove_field => ["[domain]", "[project]"] 
  }

  kv { source => "_source" }
}
<commit_msg>Hermes keystone (#227)<commit_after>input {

# consume Keystone notifications
rabbitmq {
    # only for keystone
    host => "{{.Values.hermes.rabbitmq.keystone.host}}"
    password => "{{.Values.hermes.rabbitmq.keystone.password}}"
    # the remaining parameters are equal
    user => "{{.Values.hermes.rabbitmq.user}}"
    port => {{.Values.hermes.rabbitmq.port}}
    queue => "{{.Values.hermes.rabbitmq.queue_name}}"
    subscription_retry_interval_seconds => 60
    id => "logstash_hermes_keystone"
    automatic_recovery => false
  }

# consume Nova notifications
# rabbitmq {
#     host => "{{.Values.hermes_nova_rabbitmq_host}}"
#     user => "{{.Values.hermes_nova_rabbitmq_user}}"
#     password => "{{.<PASSWORD>.<PASSWORD>}}"
#     port => {{.Values.hermes_nova_rabbitmq_port}}
#     queue => "{{.Values.hermes_nova_rabbitmq_queue_name}}"
#     subscription_retry_interval_seconds => 60
#     id => "logstash_hermes_nova"
#     automatic_recovery => false
#   }
# }
}


filter {
  # Strip oslo envelope
  mutate {
    remove_field => [ "publisher_id", "event_type", "message_id", "priority", "timestamp" ] 
  }
  ruby {
    code => "
      event.get('payload').each {|k, v|
        event.set(k, v)
      }
      event.remove('payload')
    "
  }

  # KEYSTONE TRANSFORMATIONS

  mutate {
     gsub => [
        # use proper CADF taxonomy for actions
        "action", "created\.", "create/",
        "action", "deleted\.", "delete/",
        "action", "updated\.", "create/",
        "action", "disabled\.", "disable/",
        "action", "\.", "/"
     ]
  }

  # Keystone specific transformations to compensate for scope missing from initiator element
  # When scope is missing from initiator, get it from action-specific parameters
  if ![initiator][project_id] and ![initiator][domain_id] {
    if [project] {
      mutate { rename => { "%{[project]}" => "%{[initiator][project_id]}" } }
    } else if [domain] {
      mutate { add_field => { "[initiator][domain_id]" => "%{[initiator][domain_id]}" } }
    }
  }

  # rename initiator user_id into the 'id' field for consistency
  if [initiator][user_id] {
    mutate {
      replace => { "[initiator][id]" => "%{[initiator][user_id]}" } 
      remove_field => ["[initiator][user_id]"]
    }
  }

  # add target projects/domains as attachment
  if [project] {
    ruby {
      code => "
       attachments = event.get('[target][attachments]')
        if attachments.nil?
          attachments = []
        end
        attachments << { 'name' => 'project_id', 'contentType' => '/data/security/project', 'content' => event.get('project') }
        event.set('[target][attachments]', attachments)
      "
    }
  } else if [domain] {
    ruby {
      code => "
        attachments = event.get('[target][attachments]')
        if attachments.nil?
          attachments = []
        end
        attachments << { 'name' => 'domain_id', 'contentType' => '/data/security/domain', 'content' => event.get('domain') }
        event.set('[target][attachments]', attachments)
      "
    }
  }
  if [role] {
    ruby {
      code => "
       attachments = event.get('[target][attachments]')
        if attachments.nil?
          attachments = []
        end
        attachments << { 'name' => 'role_id', 'contentType' => '/data/security/role', 'content' => event.get('role') }
        event.set('[target][attachments]', attachments)
      "
    }
  }

  # replace target ID with real user ID
  if [target][typeURI] == "service/security/account/user" and [user] {
     mutate {
       replace => { "[target][id]" => "%{[user]}" }
       remove_field => ["[user]"]
     }
  }

  # Calculate the variable index name part from payload (@metadata will not be part of the event)

  # primary index
  if [initiator][project_id] {
    mutate { add_field => { "[@metadata][index]" => "%{[initiator][project_id]}" } }
  } else if [initiator][domain_id] {
    mutate { add_field => { "[@metadata][index]" => "%{[initiator][domain_id]}" } }
  }

  # secondary index (keystone only)
  if [project] {
    mutate { add_field => { "[@metadata][index2]" => "%{[project]}" } }
  } else if [domain] {
    mutate { add_field => { "[@metadata][index2]" => "%{[domain]}" } }
  }

  # remove keystone specific fields after they have been mapped to standard attachments 
  mutate {
    remove_field => ["[domain]", "[project]"] 
  }

  kv { source => "_source" }
}

{'fraction_non_alphanumeric': 0.1721361473709894, 'fraction_numerical': 0.001829937782115408, 'mean_word_length': 2.960386473429952, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6056', 'n_tokens_mistral': 2802, 'n_tokens_neox': 2624, 'n_words': 547}