<commit_before>module(..., package.seeall)

local bt = require("apps.lwaftr.binding_table")
local constants = require("apps.lwaftr.constants")
local dump = require('apps.lwaftr.dump')
local icmp = require("apps.lwaftr.icmp")
local lwconf = require("apps.lwaftr.conf")
local lwdebug = require("apps.lwaftr.lwdebug")
local lwheader = require("apps.lwaftr.lwheader")
local lwutil = require("apps.lwaftr.lwutil")

local channel = require("apps.lwaftr.channel")
local messages = require("apps.lwaftr.messages")

local checksum = require("lib.checksum")
local ethernet = require("lib.protocol.ethernet")
local ipv6 = require("lib.protocol.ipv6")
local ipv4 = require("lib.protocol.ipv4")
local shm = require("core.shm")
local counter = require("core.counter")
local packet = require("core.packet")
local lib = require("core.lib")
local bit = require("bit")
local ffi = require("ffi")

local band, bor, bnot = bit.band, bit.bor, bit.bnot
local rshift, lshift = bit.rshift, bit.lshift
local cast = ffi.cast
local receive, transmit = link.receive, link.transmit
local rd16, wr16, rd32, ipv6_equals = lwutil.rd16, lwutil.wr16, lwutil.rd32, lwutil.ipv6_equals
local is_ipv4, is_ipv6 = lwutil.is_ipv4, lwutil.is_ipv6
local get_ihl_from_offset = lwutil.get_ihl_from_offset
local htons, htonl = lwutil.htons, lwutil.htonl
local ntohs, ntohl = htons, htonl
local keys = lwutil.keys
local write_eth_header, write_ipv6_header = lwheader.write_eth_header, lwheader.write_ipv6_header

local debug = false

-- Local bindings for constants that are used in the hot path of the
-- data plane.  Not having them here is a 1-2% performance penalty.
local ethernet_header_size = constants.ethernet_header_size
local n_ethertype_ipv4 = constants.n_ethertype_ipv4
local n_ethertype_ipv6 = constants.n_ethertype_ipv6

-- COUNTERS
-- The lwAFTR counters all live in the same directory, and their filenames are
-- built out of ordered field values, separated by dashes.
-- Fields:
-- - direction: "in", "out", "hairpin", "drop";
-- If "direction" is "drop":
--   - reason: reasons for dropping;
-- - protocol+version: "icmpv4", "icmpv6", "ipv4", "ipv6";
-- - size: "bytes", "packets".
-- TODO: should the "drop-all-ipv4/6" aggregate counters be incremented all
--   over the place, with the related runtime cost, or computed by summing
--   the relevant counters less frequently or upon access?
local counters_dir = "app/lwaftr/counters/"
-- Referenced by program/check/check.lua
counter_names = {

-- Ingress.
   "in-ipv4-bytes",
   "in-ipv4-packets",
   "in-ipv6-bytes",
   "in-ipv6-packets",

-- Egress IP.
   "out-ipv4-bytes",
   "out-ipv4-packets",
   "out-ipv6-bytes",
   "out-ipv6-packets",

-- Egress ICMP.
   "out-icmpv4-bytes",
   "out-icmpv4-packets",
   "out-icmpv6-bytes",
   "out-icmpv6-packets",

-- Hairpinning.
   "hairpin-ipv4-bytes",
   "hairpin-ipv4-packets",

-- Drop v4.

-- All dropped v4. TODO: implement.
   "drop-all-ipv4-bytes",
   "drop-all-ipv4-packets",
-- On IPv4 link, but not IPv4.
   "drop-misplaced-ipv4-bytes",
   "drop-misplaced-ipv4-packets",
-- No matching destination softwire.
   "drop-no-dest-softwire-ipv4-bytes",
   "drop-no-dest-softwire-ipv4-packets",
-- TTL is zero.
   "drop-ttl-zero-ipv4-bytes",
   "drop-ttl-zero-ipv4-packets",
-- Big packets exceeding MTU, but DF (Don't Fragment) flag set.
   "drop-over-mtu-but-dont-fragment-ipv4-bytes",
   "drop-over-mtu-but-dont-fragment-ipv4-packets",
-- Bad checksum.
   "drop-bad-checksum-icmpv4-bytes",
   "drop-bad-checksum-icmpv4-packets",
-- Policy of dropping incoming ICMPv4 packets.
   "drop-in-by-policy-icmpv4-bytes",
   "drop-in-by-policy-icmpv4-packets",
-- Policy of dropping outgoing ICMPv4 packets.
   "drop-out-by-policy-icmpv4-bytes",
   "drop-out-by-policy-icmpv4-packets",

-- Drop v6.

-- All dropped v6. TODO: implement.
   "drop-all-ipv6-bytes",
   "drop-all-ipv6-packets",
-- On IPv6 link, but not IPv6.
   "drop-misplaced-ipv6-bytes",
   "drop-misplaced-ipv6-packets",
-- Unknown IPv6 protocol.
   "drop-unknown-protocol-ipv6-bytes",
   "drop-unknown-protocol-ipv6-packets",
-- No matching source softwire.
   "drop-no-source-softwire-ipv6-bytes",
   "drop-no-source-softwire-ipv6-packets",
-- Unknown ICMPv6 type.
   "drop-unknown-protocol-icmpv6-bytes",
   "drop-unknown-protocol-icmpv6-packets",
-- "Packet too big" ICMPv6 type but not code.
   "drop-too-big-type-but-not-code-icmpv6-bytes",<commit_msg>Remove TODOs for drop-all counters<commit_after>module(..., package.seeall)

local bt = require("apps.lwaftr.binding_table")
local constants = require("apps.lwaftr.constants")
local dump = require('apps.lwaftr.dump')
local icmp = require("apps.lwaftr.icmp")
local lwconf = require("apps.lwaftr.conf")
local lwdebug = require("apps.lwaftr.lwdebug")
local lwheader = require("apps.lwaftr.lwheader")
local lwutil = require("apps.lwaftr.lwutil")

local channel = require("apps.lwaftr.channel")
local messages = require("apps.lwaftr.messages")

local checksum = require("lib.checksum")
local ethernet = require("lib.protocol.ethernet")
local ipv6 = require("lib.protocol.ipv6")
local ipv4 = require("lib.protocol.ipv4")
local shm = require("core.shm")
local counter = require("core.counter")
local packet = require("core.packet")
local lib = require("core.lib")
local bit = require("bit")
local ffi = require("ffi")

local band, bor, bnot = bit.band, bit.bor, bit.bnot
local rshift, lshift = bit.rshift, bit.lshift
local cast = ffi.cast
local receive, transmit = link.receive, link.transmit
local rd16, wr16, rd32, ipv6_equals = lwutil.rd16, lwutil.wr16, lwutil.rd32, lwutil.ipv6_equals
local is_ipv4, is_ipv6 = lwutil.is_ipv4, lwutil.is_ipv6
local get_ihl_from_offset = lwutil.get_ihl_from_offset
local htons, htonl = lwutil.htons, lwutil.htonl
local ntohs, ntohl = htons, htonl
local keys = lwutil.keys
local write_eth_header, write_ipv6_header = lwheader.write_eth_header, lwheader.write_ipv6_header

local debug = false

-- Local bindings for constants that are used in the hot path of the
-- data plane.  Not having them here is a 1-2% performance penalty.
local ethernet_header_size = constants.ethernet_header_size
local n_ethertype_ipv4 = constants.n_ethertype_ipv4
local n_ethertype_ipv6 = constants.n_ethertype_ipv6

-- COUNTERS
-- The lwAFTR counters all live in the same directory, and their filenames are
-- built out of ordered field values, separated by dashes.
-- Fields:
-- - direction: "in", "out", "hairpin", "drop";
-- If "direction" is "drop":
--   - reason: reasons for dropping;
-- - protocol+version: "icmpv4", "icmpv6", "ipv4", "ipv6";
-- - size: "bytes", "packets".
local counters_dir = "app/lwaftr/counters/"
-- Referenced by program/check/check.lua
counter_names = {

-- Ingress.
   "in-ipv4-bytes",
   "in-ipv4-packets",
   "in-ipv6-bytes",
   "in-ipv6-packets",

-- Egress IP.
   "out-ipv4-bytes",
   "out-ipv4-packets",
   "out-ipv6-bytes",
   "out-ipv6-packets",

-- Egress ICMP.
   "out-icmpv4-bytes",
   "out-icmpv4-packets",
   "out-icmpv6-bytes",
   "out-icmpv6-packets",

-- Hairpinning.
   "hairpin-ipv4-bytes",
   "hairpin-ipv4-packets",

-- Drop v4.

-- All dropped v4.
   "drop-all-ipv4-bytes",
   "drop-all-ipv4-packets",
-- On IPv4 link, but not IPv4.
   "drop-misplaced-ipv4-bytes",
   "drop-misplaced-ipv4-packets",
-- No matching destination softwire.
   "drop-no-dest-softwire-ipv4-bytes",
   "drop-no-dest-softwire-ipv4-packets",
-- TTL is zero.
   "drop-ttl-zero-ipv4-bytes",
   "drop-ttl-zero-ipv4-packets",
-- Big packets exceeding MTU, but DF (Don't Fragment) flag set.
   "drop-over-mtu-but-dont-fragment-ipv4-bytes",
   "drop-over-mtu-but-dont-fragment-ipv4-packets",
-- Bad checksum.
   "drop-bad-checksum-icmpv4-bytes",
   "drop-bad-checksum-icmpv4-packets",
-- Policy of dropping incoming ICMPv4 packets.
   "drop-in-by-policy-icmpv4-bytes",
   "drop-in-by-policy-icmpv4-packets",
-- Policy of dropping outgoing ICMPv4 packets.
   "drop-out-by-policy-icmpv4-bytes",
   "drop-out-by-policy-icmpv4-packets",

-- Drop v6.

-- All dropped v6.
   "drop-all-ipv6-bytes",
   "drop-all-ipv6-packets",
-- On IPv6 link, but not IPv6.
   "drop-misplaced-ipv6-bytes",
   "drop-misplaced-ipv6-packets",
-- Unknown IPv6 protocol.
   "drop-unknown-protocol-ipv6-bytes",
   "drop-unknown-protocol-ipv6-packets",
-- No matching source softwire.
   "drop-no-source-softwire-ipv6-bytes",
   "drop-no-source-softwire-ipv6-packets",
-- Unknown ICMPv6 type.
   "drop-unknown-protocol-icmpv6-bytes",
   "drop-unknown-protocol-icmpv6-packets",
-- "Packet too big" ICMPv6 type but not code.
   "drop-too-big-type-but-not-code-icmpv6-bytes",
{'fraction_non_alphanumeric': 0.14865022788360407, 'fraction_numerical': 0.020801682832768495, 'mean_word_length': 3.9268854346574553, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6407762', 'n_tokens_mistral': 3321, 'n_tokens_neox': 3323, 'n_words': 682}