<commit_before>--- IO stats for devices
local mod = {}

local phobos    = require "phobos"
local dpdk      = require "dpdk"
local device    = require "device"
local log       = require "log"
local colors    = require "colors"

function mod.average(data)
	local sum = 0
	for i, v in ipairs(data) do
		sum = sum + v
	end
	return sum / #data
end

function mod.median(data)
	return mod.percentile(data, 50)
end

function mod.percentile(data, p)
	local sortedData = { }
	for k, v in ipairs(data) do
		sortedData[k] = v
	end
	table.sort(sortedData)
	return sortedData[math.ceil(#data * p / 100)]
end

function mod.stdDev(data)
	local avg = mod.average(data)
	local sum = 0
	for i, v in ipairs(data) do
		sum = sum + (v - avg) ^ 2
	end
	return (sum / (#data - 1)) ^ 0.5
end

function mod.sum(data)
	local sum = 0
	for i, v in ipairs(data) do
		sum = sum + v
	end
	return sum
end

function mod.addStats(data, ignoreFirstAndLast)
	local copy = { }
	if ignoreFirstAndLast then
		for i = 2, #data - 1 do
			copy[i - 1] = data[i]
		end
	else
		for i = 1, #data do
			copy[i] = data[i]
		end
	end
	data.avg = mod.average(copy)
	data.stdDev = mod.stdDev(copy)
	data.median = mod.median(copy)
	data.sum = mod.sum(copy)
end

local colors = {
	RX = "cyan",
	TX = "blue"
}
local function getPlainUpdate(direction)
	return function(stats, file, total, mpps, mbit, wireMbit)
		file:write(("%s[%s] %s%s: %.2f Mpps, %.0f Mbit/s (%.0f Mbit/s with framing)\n"):format(
			getColorCode(colors[direction]), stats.name, direction, getColorCode(),
			mpps, mbit, wireMbit
		))
		file:flush()
	end
end

local function getPlainFinal(direction)
	return function(stats, file)
		file:write(("[%s] %s: %.2f (StdDev %.2f) Mpps, %.0f (StdDev %.0f) Mbit/s (%.0f Mbit/s with framing), total %d packets with %d bytes (incl. CRC)\n"):format(
			stats.name, direction,
			stats.mpps.avg, stats.mpps.stdDev,
			stats.mbit.avg, stats.mbit.stdDev,
			stats.wireMbit.avg,
			stats.total, stats.totalBytes<commit_msg>fix colors for final print<commit_after>--- IO stats for devices
local mod = {}

local phobos    = require "phobos"
local dpdk      = require "dpdk"
local device    = require "device"
local log       = require "log"
local colors    = require "colors"

function mod.average(data)
	local sum = 0
	for i, v in ipairs(data) do
		sum = sum + v
	end
	return sum / #data
end

function mod.median(data)
	return mod.percentile(data, 50)
end

function mod.percentile(data, p)
	local sortedData = { }
	for k, v in ipairs(data) do
		sortedData[k] = v
	end
	table.sort(sortedData)
	return sortedData[math.ceil(#data * p / 100)]
end

function mod.stdDev(data)
	local avg = mod.average(data)
	local sum = 0
	for i, v in ipairs(data) do
		sum = sum + (v - avg) ^ 2
	end
	return (sum / (#data - 1)) ^ 0.5
end

function mod.sum(data)
	local sum = 0
	for i, v in ipairs(data) do
		sum = sum + v
	end
	return sum
end

function mod.addStats(data, ignoreFirstAndLast)
	local copy = { }
	if ignoreFirstAndLast then
		for i = 2, #data - 1 do
			copy[i - 1] = data[i]
		end
	else
		for i = 1, #data do
			copy[i] = data[i]
		end
	end
	data.avg = mod.average(copy)
	data.stdDev = mod.stdDev(copy)
	data.median = mod.median(copy)
	data.sum = mod.sum(copy)
end

local colors = {
	RX = "cyan",
	TX = "blue"
}
local function getPlainUpdate(direction)
	return function(stats, file, total, mpps, mbit, wireMbit)
		file:write(("%s[%s] %s%s: %.2f Mpps, %.0f Mbit/s (%.0f Mbit/s with framing)\n"):format(
			getColorCode(colors[direction]), stats.name, direction, getColorCode(),
			mpps, mbit, wireMbit
		))
		file:flush()
	end
end

local function getPlainFinal(direction)
	return function(stats, file)
		file:write(("%s[%s] %s%s: %.2f (StdDev %.2f) Mpps, %.0f (StdDev %.0f) Mbit/s (%.0f Mbit/s with framing), total %d packets with %d bytes (incl. CRC)\n"):format(
			getColorCode(colors[direction]), stats.name, direction, getColorCode(),
			stats.mpps.avg, stats.mpps.stdDev,
			stats.mbit.avg, stats.mbit.stdDev,
			stats.wireMbit.avg,
			stats.total, stats.totalBytes
{'fraction_non_alphanumeric': 0.13972055888223553, 'fraction_numerical': 0.011976047904191617, 'mean_word_length': 3.4893617021276597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4388266', 'n_tokens_mistral': 1679, 'n_tokens_neox': 1561, 'n_words': 457}