<commit_before>-- only initialize the recent_words table once
if recent_words == nil then
	recent_words = {}
end

function dump_recent_words(channel, send_to)
	if recent_words[channel] == nil or #(recent_words[channel]) < 1 then
		irc_msg(send_to, "no recent words for "..channel)
	else
		local str = "recent words: "
		for i,v in pairs(recent_words[channel]) do
			str = str..v.." "
		end
		irc_msg(send_to, str)
	end
end

-- returns a random word from that channel's recent_words table and make sure it can be used to make a sentence
function get_recent_word(channel)
	if recent_words[channel] == nil then
		return nil
	else
		local word = nil
		local attempts = 0
		while word == nil and attempts < #(recent_words[channel]) do
			local test = recent_words[channel][math.random(1,#(recent_words[channel]))]
			local rows = sql_query_fetch(
				"SELECT `Word1`,`Word2`,`Word3` FROM `dictionary` WHERE (`Word3` != '') AND "..
				"(`Word1`='"..sql_escape(test).."' OR `Word2`='"..sql_escape(test).."' OR `Word3`='"..sql_escape(test).."') "..
				" ORDER BY RAND() LIMIT 0,1"
			)
			if #rows > 0 then
				word = test
			end
		end
		return word
	end
end

-- parses incoming messages to populate dictionary
function talk_parse(event, origin, params)
	local word1,word2
	local words = str_split(params[2], " ")
	for i=1,(#words+1) do
		local word = words[i]
		
		-- add to recent_words if above threshold
		if word ~= nil and word:len() >= 7 then
			if recent_words[params[1]] == nil then
				recent_words[params[1]] = {}
			end
			recent_words[params[1]][#(recent_words[params[1]])+1] = word
			if #(recent_words[params[1]]) > 32 then
				table.remove(recent_words[params[1]], 1)
			end
		end
		
		-- add to dictionary
		if word1 ~= nil then
			if word2 ~= nil then
				local temp = word
				if temp == nil then
					temp = ""
				end
				sql_fquery(
					"INSERT INTO `dictionary` (`Word1`, `Word2`, `Word3`, `DateAdded`) " ..
					"VALUES('"..sql_escape(word2).."','"..sql_escape(word1).."','"..sql_escape(temp).."','"..os.time().."')"
				)
			end
			word2 = word1		
		end<commit_msg>make recent_words tables shorter<commit_after>-- only initialize the recent_words table once
if recent_words == nil then
	recent_words = {}
end

-- send list of recent words from a channel
function dump_recent_words(channel, send_to)
	if recent_words[channel] == nil or #(recent_words[channel]) < 1 then
		irc_msg(send_to, "no recent words for "..channel)
	else
		local str = "recent words: "
		for i,v in pairs(recent_words[channel]) do
			str = str..v.." "
		end
		irc_msg(send_to, str)
	end
end

-- returns a random word from that channel's recent_words table and make sure it can be used to make a sentence
function get_recent_word(channel)
	if recent_words[channel] == nil then
		return nil
	else
		local word = nil
		local attempts = 0
		while word == nil and attempts < #(recent_words[channel]) do
			local test = recent_words[channel][math.random(1,#(recent_words[channel]))]
			local rows = sql_query_fetch(
				"SELECT `Word1`,`Word2`,`Word3` FROM `dictionary` WHERE (`Word3` != '') AND "..
				"(`Word1`='"..sql_escape(test).."' OR `Word2`='"..sql_escape(test).."' OR `Word3`='"..sql_escape(test).."') "..
				" ORDER BY RAND() LIMIT 0,1"
			)
			if #rows > 0 then
				word = test
			end
		end
		return word
	end
end

-- parses incoming messages to populate dictionary
function talk_parse(event, origin, params)
	local word1,word2
	local words = str_split(params[2], " ")
	for i=1,(#words+1) do
		local word = words[i]
		
		-- add to recent_words if above threshold
		if word ~= nil and word:len() >= 8 then
			if recent_words[params[1]] == nil then
				recent_words[params[1]] = {}
			end
			recent_words[params[1]][#(recent_words[params[1]])+1] = word
			if #(recent_words[params[1]]) > 12 then
				table.remove(recent_words[params[1]], 1)
			end
		end
		
		-- add to dictionary
		if word1 ~= nil then
			if word2 ~= nil then
				local temp = word
				if temp == nil then
					temp = ""
				end
				sql_fquery(
					"INSERT INTO `dictionary` (`Word1`, `Word2`, `Word3`, `DateAdded`) " ..
					"VALUES('"..sql_escape(word2).."','"..sql_escape(word1).."','"..sql_escape(temp).."','"..os.time().."')"
				)
			end
			word2 = word1		
		end
{'fraction_non_alphanumeric': 0.1511930073234113, 'fraction_numerical': 0.01795416961965509, 'mean_word_length': 3.4804232804232806, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6132118', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1465, 'n_words': 422}