<commit_before># Copyright 2017 Istio Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
#
#
# This is the global dictionary used by mixer client and server
# for sending mixer attributes.
#
# The main reason using yaml format is that it allows comments like this.
#
# Important update rules:
# * never remove, only add words at the end of file.
# * http headers should be lower case.
# * not duplication, all words have to be unique.
#
# Tools:
# 1) It can be converted to json by:
#  python -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), \
#  sys.stdout, indent=4)' < global_dictionary.yaml
#
# 2) Use this to detect uniqueness and print out duplication:
#  python -c 'import yaml, collections; print [item for item, count in \
#  collections.Counter(yaml.load(open("global_dictionary.yaml"))).items() if count > 1]'
#

# Section 1: Standard Istio attribute vocabulary
# https://istio.io/docs/reference/config/mixer/attribute-vocabulary.html

- source.ip
- source.port
- source.name
- source.uid
- source.namespace
- source.labels
- source.user
- target.ip
- target.port
- target.service
- target.name
- target.uid
- target.namespace
- target.labels
- target.user
- request.headers
- request.id
- request.path
- request.host
- request.method
- request.reason
- request.referer
- request.scheme
- request.size
- request.time
- request.useragent
- response.headers
- response.size
- response.time
- response.duration
- response.code

# Section 2: HTTP common headers extracted from
# https://tools.ietf.org/html/rfc7541#appendix-A
- :authority
- :method
- :path
- :scheme
- :status
- access-control-allow-origin
- access-control-allow-methods
- access-control-allow-headers
- access-control-max-age
- access-control-request-method
- access-control-request-headers
- accept-charset
- accept-encoding
- accept-language
- accept-ranges
- accept
- access-control-allow
- age
- allow
- authorization
- cache-control
- content-disposition
- content-encoding
- content-language
- content-length
- content-location
- content-range
- content-type
- cookie
- date
- etag
- expect
- expires
- from
- host
- if-match
- if-modified-since
- if-none-match
- if-range
- if-unmodified-since
- keep-alive
- last-modified
- link
- location
- max-forwards
- proxy-authenticate
- proxy-authorization
- range
- referer
- refresh
- retry-after
- server
- set-cookie
- strict-transport-sec
- transfer-encoding
- user-agent
- vary
- via
- www-authenticate

# HTTP methods
- GET
- POST

# Common strings
- http
- envoy
- '200'
- Keep-Alive
- chunked
- x-envoy-service-time
- x-forwarded-for
- x-forwarded-host
- x-forwarded-proto
- x-http-method-override
- x-request-id
- x-requested-with

# Popular content-type
- application/json
- application/xml
- gzip
- text/html
- text/html; charset=utf-8
- text/plain
- text/plain; charset=utf-8

# common strings
- '0'
- '1'
- true
- false
- gzip, deflate
- max-age=0
- x-envoy-upstream-service-time
- x-envoy-internal
- x-envoy-expected-rq-timeout-ms
- x-ot-span-context
- x-b3-traceid
- x-b3-sampled
- x-b3-spanid
- tcp

# connection attributes
- connection.id
- connection.received.bytes
- connection.received.bytes_total
- connection.sent.bytes
- connection.sent.bytes_total
- connection.duration

# context attributes
- context.protocol
- context.timestamp
- context.time<commit_msg>Add common integer strings. (#146)<commit_after># Copyright 2017 Istio Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
#
#
# This is the global dictionary used by mixer client and server
# for sending mixer attributes.
#
# The main reason using yaml format is that it allows comments like this.
#
# Important update rules:
# * never remove, only add words at the end of file.
# * http headers should be lower case.
# * not duplication, all words have to be unique.
#
# Tools:
# 1) It can be converted to json by:
#  python -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), \
#  sys.stdout, indent=4)' < global_dictionary.yaml
#
# 2) Use this to detect uniqueness and print out duplication:
#  python -c 'import yaml, collections; print [item for item, count in \
#  collections.Counter(yaml.load(open("global_dictionary.yaml"))).items() if count > 1]'
#

# Section 1: Standard Istio attribute vocabulary
# https://istio.io/docs/reference/config/mixer/attribute-vocabulary.html

- source.ip
- source.port
- source.name
- source.uid
- source.namespace
- source.labels
- source.user
- target.ip
- target.port
- target.service
- target.name
- target.uid
- target.namespace
- target.labels
- target.user
- request.headers
- request.id
- request.path
- request.host
- request.method
- request.reason
- request.referer
- request.scheme
- request.size
- request.time
- request.useragent
- response.headers
- response.size
- response.time
- response.duration
- response.code

# Section 2: HTTP common headers extracted from
# https://tools.ietf.org/html/rfc7541#appendix-A
- :authority
- :method
- :path
- :scheme
- :status
- access-control-allow-origin
- access-control-allow-methods
- access-control-allow-headers
- access-control-max-age
- access-control-request-method
- access-control-request-headers
- accept-charset
- accept-encoding
- accept-language
- accept-ranges
- accept
- access-control-allow
- age
- allow
- authorization
- cache-control
- content-disposition
- content-encoding
- content-language
- content-length
- content-location
- content-range
- content-type
- cookie
- date
- etag
- expect
- expires
- from
- host
- if-match
- if-modified-since
- if-none-match
- if-range
- if-unmodified-since
- keep-alive
- last-modified
- link
- location
- max-forwards
- proxy-authenticate
- proxy-authorization
- range
- referer
- refresh
- retry-after
- server
- set-cookie
- strict-transport-sec
- transfer-encoding
- user-agent
- vary
- via
- www-authenticate

# HTTP methods
- GET
- POST

# Common strings
- http
- envoy
- '200'
- Keep-Alive
- chunked
- x-envoy-service-time
- x-forwarded-for
- x-forwarded-host
- x-forwarded-proto
- x-http-method-override
- x-request-id
- x-requested-with

# Popular content-type
- application/json
- application/xml
- gzip
- text/html
- text/html; charset=utf-8
- text/plain
- text/plain; charset=utf-8

# common strings
- '0'
- '1'
- true
- false
- gzip, deflate
- max-age=0
- x-envoy-upstream-service-time
- x-envoy-internal
- x-envoy-expected-rq-timeout-ms
- x-ot-span-context
- x-b3-traceid
- x-b3-sampled
- x-b3-spanid
- tcp

# connection attributes
- connection.id
- connection.received.bytes
- connection.received.bytes_total
- connection.sent.bytes
- connection.sent.bytes_total
- connection.duration

# context attributes
- context.protocol
- context.timestamp
- context.time

# common integer strings
- 0
- 1
- 200
- 302
- 400
- 401
- 403
- 404
- 409
- 429
- 499
- 500
- 501
- 502
- 503
- 504
{'fraction_non_alphanumeric': 0.13534493552022037, 'fraction_numerical': 0.013146362839614373, 'mean_word_length': 3.4108227498619548, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1532683', 'n_tokens_mistral': 2629, 'n_tokens_neox': 2510, 'n_words': 720}