<commit_before>local histogram = {}
histogram.__index = histogram

function histogram.create()
	local histo = {}
	setmetatable(histo, histogram)
	histo.histo = {}
	histo.dirty = true
	return histo
end

function histogram:update(k)
	self.histo[k] = (self.histo[k] or 0) +1
	self.dirty = true
end

function histogram:calc()
	self.sortedHisto = {}
	self.sum = 0
	self.samples = 0

	for k, v in pairs(self.histo) do
		table.insert(self.sortedHisto, { k = k, v = v})
		self.samples = self.samples + v
		self.sum = self.sum + k * v
	end
	self.avg = self.sum / self.samples
	table.sort(self.sortedHisto, function(e1, e2) return e1.k < e2.k end)
	
	local quartSamples = self.samples / 4

	self._lowerQuart = nil
	self._median = nil
	self._upperQuart = nil

	local idx = 0
	for _, p in ipairs(self.sortedHisto) do
		if not self._lowerQuart and idx >= quartSamples then
			self._lowerQuart = p.k
		elseif not self._median and idx >= quartSamples * 2 then
			self._median = p.k
		elseif not self._upperQuart and idx >= quartSamples * 3 then
			self._upperQuart = p.k
			break
		end
		idx = idx + p.v
	end
	self.dirty = false
end

function histogram:quartiles()
	if self.dirty then self:calc() end

	return self._lowerQuart, self._median, self._upperQuart
end

function histogram:samples()
	local i = 0
	if self.dirty then self:calc() end
	local n = #(self.sortedHisto)
	return function ()
		if not self.dirty then
			i = i+1
			if i <= n then return self.sortedHisto[i] end
		end
	end
end

return histogram<commit_msg>more formatting<commit_after>local histogram = {}
histogram.__index = histogram

function histogram.create()
	local histo = {}
	setmetatable(histo, histogram)
	histo.histo = {}
	histo.dirty = true
	return histo
end

function histogram:update(k)
	self.histo[k] = (self.histo[k] or 0) +1
	self.dirty = true
end

function histogram:calc()
	self.sortedHisto = {}
	self.sum = 0
	self.samples = 0

	for k, v in pairs(self.histo) do
		table.insert(self.sortedHisto, {k = k, v = v})
		self.samples = self.samples + v
		self.sum = self.sum + k * v
	end
	self.avg = self.sum / self.samples
	table.sort(self.sortedHisto, function(e1, e2) return e1.k < e2.k end)
	
	local quartSamples = self.samples / 4

	self.lowerQuart = nil
	self.median = nil
	self.upperQuart = nil

	local idx = 0
	for _, p in ipairs(self.sortedHisto) do
		if not self.lowerQuart and idx >= quartSamples then
			self.lowerQuart = p.k
		elseif not self.median and idx >= quartSamples * 2 then
			self.median = p.k
		elseif not self.upperQuart and idx >= quartSamples * 3 then
			self.upperQuart = p.k
			break
		end
		idx = idx + p.v
	end
	self.dirty = false
end

function histogram:quartiles()
	if self.dirty then self:calc() end

	return self.lowerQuart, self.median, self.upperQuart
end

function histogram:samples()
	local i = 0
	if self.dirty then self:calc() end
	local n = #self.sortedHisto
	return function()
		if not self.dirty then
			i = i + 1
			if i <= n then return self.sortedHisto[i] end
		end
	end
end

return histogram
{'fraction_non_alphanumeric': 0.1072618254497002, 'fraction_numerical': 0.009327115256495669, 'mean_word_length': 3.416176470588235, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6855197', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1089, 'n_words': 327}