<commit_before>-- vi: set ft=lua :

extended_custom = format {
  -- Netscape Common
  -- chi: Client: IP address of the client’s host.
  -- caun: Client Request: Authentication User name
  -- cqtn: Client Request: Client request timestamp
  -- cqtx: Client Request: HTTP client request text
  -- pssc: Proxy Response: HTTP response status code sent by Traffic Server proxy to the client.
  -- pscl: Proxy Response: Content body length of the Traffic Server proxy response.
  --
  -- Netscape Extended-2
  -- sssc: Origin Response: HTTP response status code sent by the origin server to the Traffic Server proxy.
  -- sscl: Origin Response: Content body length of the origin server response to Traffic Server.
  -- cqcl: Client Request: Client request content length, in bytes.
  -- pqcl: Proxy Request: Content body length of the Traffic Server proxy request to the origin server.
  -- pqcl: Proxy Request: Content body length of the Traffic Server proxy request to the origin server.
  -- cqhl: Client Request: Client request header length, in bytes.
  -- pshl: Proxy Response: Header length of the Traffic Server response to client.
  -- pqhl: Proxy Request: Header length of the Traffic Server proxy request to the origin server.
  -- sshl: Origin Response: Header length of the origin server response.
  -- ttms: Client-Proxy Connection: Time in milliseconds spent by Traffic Server processing the entire client request. Measured from the time the connection between the client and Traffic Server proxy was established until the last byte of the proxy response was delivered to the client.
  -- stms: Proxy-Origin Connection: Time (in milliseconds) spent accessing the origin server. Measured from the time the connection between proxy and origin is established to the time it was closed.
  -- phr: Proxy: Proxy Hierarchy Route. Specifies the route through configured hierarchical caches used to retrieve the object.
  -- cfsc: Client Request: Finish status code specifying whether the client request to Traffic Server was successfully completed (FIN) or interrupted (INTR).
  -- pfsc: Proxy Request: Finish status code specifying whether the proxy request from Traffic Server to the origin server was successfully completed (FIN), interrupted (INTR), or timed out (TIMEOUT).
  -- crc: Proxy Cache: Cache Result Code. The result of Traffic Server attempting to obtain the object from cache.
  --
  -- Custom
  -- chm: Proxy Cache: Cache Hit-Miss status. Specifies the level of cache from which this request was served by Traffic Server. Currently supports only RAM (HIT_RAM) vs disk (HIT_DISK).
  -- cwr: Proxy Cache: Cache Write Result. Specifies the result of attempting to write to cache: not relevant (-), no cache write (WL_MISS), write interrupted (INTR), error while writing (ERR), or cache write successful (FIN).
  -- sca: Proxy: Number of attempts within the current transaction by Traffic Server in connecting to the origin server.
  -- sstc: Proxy: Number of transactions between the Traffic Server proxy and the origin server from a single session. Any value greater than zero indicates connection reuse.
  -- psct: Origin Server: Response Content type of the document obtained by Traffic Server from the origin server response.
  -- shi: Origin Server: IP address resolved via DNS by Traffic Server for the origin server. For hosts with multiple IP addresses, the address used by Traffic Server for the connection will be reported.
  -- shn: Origin Server: Host name of the origin server.
  -- pqsp: Proxy Request: Port number from which Traffic Server issued the proxy request to the origin server. Cache hits will yield a value of 0.
  -- cqhv: Client Request: Client request HTTP version.
  -- sshv: Origin Response: Origin server’s response HTTP version.
  -- pqssl: Proxy Request: Indicates whether the connection from Traffic Server to the origin was over SSL or not.
  -- csssc: Cached Proxy: Response HTTP response status code of the origin server response, as cached by Traffic Server.
  -- cqtr: Client Request: TCP reused status of the connection between the client and Traffic Server proxy, indicating whether the request was delivered through an already established connection.
  -- cquuc: Client Request: Canonical (prior to remapping) effective URL from client request.
  Format = '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> id=%<{X-Api-Umbrella-Request-ID}cqh> sssc=%<sssc> sscl=%<sscl> cqcl=%<cqcl> pqcl=%<pqcl> cqhl=%<cqhl> pshl=%<pshl> pqhl=%<pqhl> sshl=%<sshl> ttms=%<ttms> stms=%<stms> phr=%<phr> cfsc=%<cfsc> pfsc=%<pfsc> crc=%<crc> chm=%<chm> cwr=%<cwr> sca=%<sca> sstc=%<sstc> psct="%<psct>" shi=%<shi> shn=%<shn> pqsp=%<pqsp> cqhv=%<cqhv> sshv=%<sshv> pqssl=%<pqssl> csssc=%<csssc> cqtr=%<cqtr> cquuc="%<cquuc>" muabms=%<{TS_MILESTONE_UA_BEGIN-TS_MILESTONE_SM_START}msdms> muarhdms=%<{TS_MILESTONE_UA_READ_HEADER_DONE-TS_MILESTONE_SM_START}msdms> muabwms=%<{TS_MILESTONE_UA_BEGIN_WRITE-TS_MILESTONE_SM_START}msdms> muacms=%<{TS_MILESTONE_UA_CLOSE-TS_MILESTONE_SM_START}msdms> msfcms=%<{TS_MILESTONE_SERVER_FIRST_CONNECT-TS_MILESTONE_SM_START}msdms> mscms=%<{TS_MILESTONE_SERVER_CONNECT-TS_MILESTONE_SM_START}msdms> mscems=%<{TS_MILESTONE_SERVER_CONNECT_END-TS_MILESTONE_SM_START}msdms> msbwms=%<{TS_MILESTONE_SERVER_BEGIN_WRITE-TS_MILESTONE_SM_START}msdms> msfrms=%<{TS_MILESTONE_SERVER_FIRST_READ-TS_MILESTONE_SM_START}msdms> msrhdms=%<{TS_MILESTONE_SERVER_READ_HEADER_DONE-TS_MILESTONE_SM_START}msdms> mscms=%<{TS_MILESTONE_SERVER_CLOSE-TS_MILESTONE_SM_START}msdms> mcorbms=%<{TS_MILESTONE_CACHE_OPEN_READ_BEGIN-TS_MILESTONE_SM_START}msdms> mcorems=%<{TS_MILESTONE_CACHE_OPEN_READ_END-TS_MILESTONE_SM_START}msdms> mcowbms=%<{TS_MILESTONE_CACHE_OPEN_WRITE_BEGIN-TS_MILESTONE_SM_START}msdms> mcowems=%<{TS_MILESTONE_CACHE_OPEN_WRITE_END-TS_MILESTONE_SM_START}msdms> mdlbms=%<{TS_MILESTONE_DNS_LOOKUP_BEGIN-TS_MILESTONE_SM_START}msdms> mdlems=%<{TS_MILESTONE_DNS_LOOKUP_END-TS_MILESTONE_SM_START}msdms> msmsms=%<{TS_MILESTONE_SM_START-TS_MILESTONE_SM_START}msdms> msmfms=%<{TS_MILESTONE_SM_FINISH-TS_MILESTONE_SM_START}msdms> mpams=%<{TS_MILESTONE_PLUGIN_ACTIVE-TS_MILESTONE_SM_START}msdms> mptms=%<{TS_MILESTONE_PLUGIN_TOTAL-TS_MILESTONE_SM_START}msdms>'
}

log.ascii {
  Format = extended_custom,
  Filename = 'access.log'
}<commit_msg>Fix duplicate "mscms" attribute in trafficserver log.<commit_after>-- vi: set ft=lua :

extended_custom = format {
  -- Netscape Common
  -- chi: Client: IP address of the client’s host.
  -- caun: Client Request: Authentication User name
  -- cqtn: Client Request: Client request timestamp
  -- cqtx: Client Request: HTTP client request text
  -- pssc: Proxy Response: HTTP response status code sent by Traffic Server proxy to the client.
  -- pscl: Proxy Response: Content body length of the Traffic Server proxy response.
  --
  -- Netscape Extended-2
  -- sssc: Origin Response: HTTP response status code sent by the origin server to the Traffic Server proxy.
  -- sscl: Origin Response: Content body length of the origin server response to Traffic Server.
  -- cqcl: Client Request: Client request content length, in bytes.
  -- pqcl: Proxy Request: Content body length of the Traffic Server proxy request to the origin server.
  -- pqcl: Proxy Request: Content body length of the Traffic Server proxy request to the origin server.
  -- cqhl: Client Request: Client request header length, in bytes.
  -- pshl: Proxy Response: Header length of the Traffic Server response to client.
  -- pqhl: Proxy Request: Header length of the Traffic Server proxy request to the origin server.
  -- sshl: Origin Response: Header length of the origin server response.
  -- ttms: Client-Proxy Connection: Time in milliseconds spent by Traffic Server processing the entire client request. Measured from the time the connection between the client and Traffic Server proxy was established until the last byte of the proxy response was delivered to the client.
  -- stms: Proxy-Origin Connection: Time (in milliseconds) spent accessing the origin server. Measured from the time the connection between proxy and origin is established to the time it was closed.
  -- phr: Proxy: Proxy Hierarchy Route. Specifies the route through configured hierarchical caches used to retrieve the object.
  -- cfsc: Client Request: Finish status code specifying whether the client request to Traffic Server was successfully completed (FIN) or interrupted (INTR).
  -- pfsc: Proxy Request: Finish status code specifying whether the proxy request from Traffic Server to the origin server was successfully completed (FIN), interrupted (INTR), or timed out (TIMEOUT).
  -- crc: Proxy Cache: Cache Result Code. The result of Traffic Server attempting to obtain the object from cache.
  --
  -- Custom
  -- chm: Proxy Cache: Cache Hit-Miss status. Specifies the level of cache from which this request was served by Traffic Server. Currently supports only RAM (HIT_RAM) vs disk (HIT_DISK).
  -- cwr: Proxy Cache: Cache Write Result. Specifies the result of attempting to write to cache: not relevant (-), no cache write (WL_MISS), write interrupted (INTR), error while writing (ERR), or cache write successful (FIN).
  -- sca: Proxy: Number of attempts within the current transaction by Traffic Server in connecting to the origin server.
  -- sstc: Proxy: Number of transactions between the Traffic Server proxy and the origin server from a single session. Any value greater than zero indicates connection reuse.
  -- psct: Origin Server: Response Content type of the document obtained by Traffic Server from the origin server response.
  -- shi: Origin Server: IP address resolved via DNS by Traffic Server for the origin server. For hosts with multiple IP addresses, the address used by Traffic Server for the connection will be reported.
  -- shn: Origin Server: Host name of the origin server.
  -- pqsp: Proxy Request: Port number from which Traffic Server issued the proxy request to the origin server. Cache hits will yield a value of 0.
  -- cqhv: Client Request: Client request HTTP version.
  -- sshv: Origin Response: Origin server’s response HTTP version.
  -- pqssl: Proxy Request: Indicates whether the connection from Traffic Server to the origin was over SSL or not.
  -- csssc: Cached Proxy: Response HTTP response status code of the origin server response, as cached by Traffic Server.
  -- cqtr: Client Request: TCP reused status of the connection between the client and Traffic Server proxy, indicating whether the request was delivered through an already established connection.
  -- cquuc: Client Request: Canonical (prior to remapping) effective URL from client request.
  Format = '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> id=%<{X-Api-Umbrella-Request-ID}cqh> sssc=%<sssc> sscl=%<sscl> cqcl=%<cqcl> pqcl=%<pqcl> cqhl=%<cqhl> pshl=%<pshl> pqhl=%<pqhl> sshl=%<sshl> ttms=%<ttms> stms=%<stms> phr=%<phr> cfsc=%<cfsc> pfsc=%<pfsc> crc=%<crc> chm=%<chm> cwr=%<cwr> sca=%<sca> sstc=%<sstc> psct="%<psct>" shi=%<shi> shn=%<shn> pqsp=%<pqsp> cqhv=%<cqhv> sshv=%<sshv> pqssl=%<pqssl> csssc=%<csssc> cqtr=%<cqtr> cquuc="%<cquuc>" muabms=%<{TS_MILESTONE_UA_BEGIN-TS_MILESTONE_SM_START}msdms> muarhdms=%<{TS_MILESTONE_UA_READ_HEADER_DONE-TS_MILESTONE_SM_START}msdms> muabwms=%<{TS_MILESTONE_UA_BEGIN_WRITE-TS_MILESTONE_SM_START}msdms> muacms=%<{TS_MILESTONE_UA_CLOSE-TS_MILESTONE_SM_START}msdms> msfcms=%<{TS_MILESTONE_SERVER_FIRST_CONNECT-TS_MILESTONE_SM_START}msdms> mscms=%<{TS_MILESTONE_SERVER_CONNECT-TS_MILESTONE_SM_START}msdms> mscems=%<{TS_MILESTONE_SERVER_CONNECT_END-TS_MILESTONE_SM_START}msdms> msbwms=%<{TS_MILESTONE_SERVER_BEGIN_WRITE-TS_MILESTONE_SM_START}msdms> msfrms=%<{TS_MILESTONE_SERVER_FIRST_READ-TS_MILESTONE_SM_START}msdms> msrhdms=%<{TS_MILESTONE_SERVER_READ_HEADER_DONE-TS_MILESTONE_SM_START}msdms> msclms=%<{TS_MILESTONE_SERVER_CLOSE-TS_MILESTONE_SM_START}msdms> mcorbms=%<{TS_MILESTONE_CACHE_OPEN_READ_BEGIN-TS_MILESTONE_SM_START}msdms> mcorems=%<{TS_MILESTONE_CACHE_OPEN_READ_END-TS_MILESTONE_SM_START}msdms> mcowbms=%<{TS_MILESTONE_CACHE_OPEN_WRITE_BEGIN-TS_MILESTONE_SM_START}msdms> mcowems=%<{TS_MILESTONE_CACHE_OPEN_WRITE_END-TS_MILESTONE_SM_START}msdms> mdlbms=%<{TS_MILESTONE_DNS_LOOKUP_BEGIN-TS_MILESTONE_SM_START}msdms> mdlems=%<{TS_MILESTONE_DNS_LOOKUP_END-TS_MILESTONE_SM_START}msdms> msmsms=%<{TS_MILESTONE_SM_START-TS_MILESTONE_SM_START}msdms> msmfms=%<{TS_MILESTONE_SM_FINISH-TS_MILESTONE_SM_START}msdms> mpams=%<{TS_MILESTONE_PLUGIN_ACTIVE-TS_MILESTONE_SM_START}msdms> mptms=%<{TS_MILESTONE_PLUGIN_TOTAL-TS_MILESTONE_SM_START}msdms>'
}

log.ascii {
  Format = extended_custom,
  Filename = 'access.log'
}
{'fraction_non_alphanumeric': 0.08866175421915855, 'fraction_numerical': 0.0003169320972981539, 'mean_word_length': 5.439795918367347, 'pattern_counts': {'":': 0, '<': 115, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7145141', 'n_tokens_mistral': 4212, 'n_tokens_neox': 4100, 'n_words': 1368}