<commit_before>local vector = require 'lib.hump.vector'

-- common query functions
-- t is guaranteed to be an array with at least one value
-- each funciton is recursive to enable operating on nested tables

local sumFunc = function(t)
	local sum
	local isTable = type(t[1]) == 'table'
	local isVector = vector.isvector(t[1])

	for k,v in pairs(t) do
		if isVector or type(v) == 'number' then
			sum = sum and sum + v or v
		elseif isTable then
			sum = sum or {}
			for key,val in pairs(v) do
				if type(val) == 'number' then
					sum[key] = sum[key] and sum[key] + val or val
				end
			end
		end
	end

	return sum
end


local meanFunc = function(t)
	local mean
	local count = 0
	local isTable = type(t[1]) == 'table'
	local isVector = vector.isvector(t[1])

	for k,v in pairs(t) do
		if isVector or type(v) == 'number' then
			mean = mean and mean + v or v
			count = count + 1
		elseif isTable then
			mean = mean or {}
			local found = false
			for key,val in pairs(v) do
				if type(val) == 'number' then
					mean[key] = mean[key] and mean[key] + val or val
					found = true
				end
			end
			if found then count = count + 1 end
		end
	end

	if isTable then
		for k in pairs(mean) do mean[k] = mean[k] / count end
	else
		mean = mean/count
	end

	return mean
end


local productFunc = function(t)
	local product
	local isTable = type(t[1]) == 'table'
	local isVector = vector.isvector(t[1])

	for k,v in pairs(t) do
		if isVector or type(v) == 'number' then
			product = product and product * v or v
		elseif isTable then
			product = product or {}
			for key,val in pairs(v) do
				if type(val) == 'number' then
					product[key] = product[key] and product[key] * val or val
				end
			end
		end
	end

	return product
end


local randomFunc = function(t) return t[Random(#t)] end
local existsFunc = function(t) return true end


return {
	sum = sumFunc,
	mean = meanFunc,
	product = productFunc,
	random = randomFunc,
	exists = existsFunc
}
<commit_msg>add boolean AND when 'sum' is queried on booleans<commit_after>local vector = require 'lib.hump.vector'

-- common query functions
-- t is guaranteed to be an array with at least one value
-- each funciton is recursive to enable operating on nested tables

local sumFunc = function(t)
	local sum
	local isTable = type(t[1]) == 'table'
	local isVector = vector.isvector(t[1])

	for k,v in pairs(t) do
		if isVector or type(v) == 'number' then
			sum = sum and sum + v or v
		elseif isTable then
			sum = sum or {}
			for key,val in pairs(v) do
				if type(val) == 'number' then
					sum[key] = sum[key] and sum[key] + val or val
				end
			end
		elseif type(v) == 'boolean' then
			if sum == nil then sum = v else sum = sum and v end
		end
	end

	return sum
end


local meanFunc = function(t)
	local mean
	local count = 0
	local isTable = type(t[1]) == 'table'
	local isVector = vector.isvector(t[1])

	for k,v in pairs(t) do
		if isVector or type(v) == 'number' then
			mean = mean and mean + v or v
			count = count + 1
		elseif isTable then
			mean = mean or {}
			local found = false
			for key,val in pairs(v) do
				if type(val) == 'number' then
					mean[key] = mean[key] and mean[key] + val or val
					found = true
				end
			end
			if found then count = count + 1 end
		end
	end

	if isTable then
		for k in pairs(mean) do mean[k] = mean[k] / count end
	else
		mean = mean/count
	end

	return mean
end


local productFunc = function(t)
	local product
	local isTable = type(t[1]) == 'table'
	local isVector = vector.isvector(t[1])

	for k,v in pairs(t) do
		if isVector or type(v) == 'number' then
			product = product and product * v or v
		elseif isTable then
			product = product or {}
			for key,val in pairs(v) do
				if type(val) == 'number' then
					product[key] = product[key] and product[key] * val or val
				end
			end
		end
	end

	return product
end


local randomFunc = function(t) return t[Random(#t)] end
local existsFunc = function(t) return true end


return {
	sum = sumFunc,
	mean = meanFunc,
	product = productFunc,
	random = randomFunc,
	exists = existsFunc
}

{'fraction_non_alphanumeric': 0.10162802170695609, 'fraction_numerical': 0.004440059200789344, 'mean_word_length': 2.9990138067061145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6510729', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1399, 'n_words': 550}