<commit_before>local iconv = require"iconv"
local iso2utf = iconv.new("utf-8", "iso-8859-15")
local utf2iso = iconv.new('iso-8859-15', 'utf-8')

return {
	["^:(%S+) PRIVMSG (%S+) :!dokpro (.+)$"] = function(self, src, dest, msg)
		local query = utils.escape(utf2iso:iconv(msg)):gsub('%s', '+')
		print(query)
		local content, status = utils.http("http://www.nob-ordbok.uio.no/perl/ordbok.cgi?ordbok=bokmaal&bokmaal=+&OPP=" .. query)

		if(content) then
			-- It might explode, but shouldn't!
			local ans = content:match('</table></td><td>(.-)</table>')
			local nick = src:match'^([^!]+)'
			if(ans) then
				ans = iso2utf:iconv(ans)
				-- strip out all the html and convert entities:
				ans = utils.decodeHTML(ans:gsub('<%/?[%w:]+.-%/?>', ''))
				self:msg(
					dest, src,
					"%s: %s | http://x0.no/dokpro/%s",
					nick,
					ans,
					msg:gsub('%s', '+')
				)
			else
				self:msg(dest, src, '%s: %s', nick, 'Du suger, prøv igjen')
			end
		end
	end
}<commit_msg>dokpro: Port over to the new core.<commit_after>local simplehttp = require'simplehttp'
local iconv = require"iconv"
local iso2utf = iconv.new("utf-8", "iso-8859-15")
local utf2iso = iconv.new('iso-8859-15', 'utf-8')

local urlEncode = function(str)
	return str:gsub(
		'([^%w ])',
		function (c)
			return string.format ("%%%02X", string.byte(c))
		end
	):gsub(' ', '+')
end

local trim = function(s)
	return s:match('^()%s*$') and '' or s:match('^%s*(.*%S)')
end

local parseData = function(data)
	data = iso2utf:iconv(data)

	-- This page is a typical example of someone using XHTML+CSS+JS, while still
	-- coding like they used to back in 1998.
	data = data:gsub('\r', ''):match('<div id="kolonne_enkel"[^>]+>(.-)</div>'):gsub('&nbsp;', '')

	local words = {}
	local entryIndex = 1
	for entryData in data:gmatch('<table class="liten_ordliste">([^\n]+)') do
		local entry = {
			lookup = {},
			examples = {},
		}

		local insertAt
		for td in entryData:gmatch('<td[^>]->(.-)</td>') do
			-- Strip away HTML.
			local line = trim(td:gsub('<span class="b">([^%d]-)</span>', '%1'):gsub('</?[%w:]+[^>]-/?>', ''))
			line = line:gsub('%s%s+', ' ')

			if(#line > 0) then
				if(tonumber(line)) then
					insertAt = tonumber(line)
				elseif(not line:match('%s+')) then
					table.insert(entry.lookup, line)
				elseif(insertAt) then
					entry.examples[insertAt] = line
					insertAt = nil
				else
					entry.meaning = line
				end
			end
		end

		table.insert(words, entry)
	end

	return words
end

local handleInput = function(self, source, destination, word)
	local query = urlEncode(utf2iso:iconv(word))
	simplehttp(
		"http://www.nob-ordbok.uio.no/perl/ordbok.cgi?ordbok=bokmaal&bokmaal=+&OPP=" .. query,

		function(data)
			local words = parseData(data)

			local msgLimit = (512 - 16 - 65 - 10) - #self.config.nick - #destination
			-- size of the word + x0 url.
			local n =  #word + 23
			local out = {}
			for i=1, #words do
				local word = words[i]
				local lookup = table.concat(word.lookup, ', ')
				local message = string.format('\002[%s]\002: %s', lookup, word.meaning)

				n = n + #message
				if(n < msgLimit) then
					table.insert(out, message)
				else
					break
				end
			end

			if(#out > 0) then
				self:Msg('privmsg', destination, source, '%s | http://x0.no/dokpro/%s', table.concat(out, ', '), word)
			else
				self:Msg('privmsg', destination, source, '%s: %s', source.nick, 'Du suger, prøv igjen.')
			end
		end
	)
end

return {
	PRIVMSG = {
		['^!dokpro (.+)$'] = handleInput,
		['^!ordbok (.+)$'] = handleInput,
	},
}
{'fraction_non_alphanumeric': 0.20646625070901872, 'fraction_numerical': 0.019001701644923426, 'mean_word_length': 3.3705080545229245, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2836513', 'n_tokens_mistral': 1613, 'n_tokens_neox': 1367, 'n_words': 307}