<commit_before>local iconv = require"iconv"
local uri = require"handler.uri"

local simplehttp = require'simplehttp'
local x0 = require'x0'
local html2unicode = require'html'
local nixio = require'nixio'

local uri_parse = uri.parse
local DL_LIMIT = 2^17

local patterns = {
	-- X://Y url
	"^(https?://%S+)",
    "^<(https?://%S+)>",
	"%f[%S](https?://%S+)",
	-- www.X.Y url
	"^(www%.[%w_-%%]+%.%S+)",
	"%f[%S](www%.[%w_-%%]+%.%S+)",
}

local translateCharset = {
	utf8 = 'utf-8',
	['x-sjis'] = 'sjis',
	['ks_c_5601-1987'] = 'euc-kr',
	['ksc_5601'] = 'euc-kr',
}

local trim = function(str)
	return str:match('^%s*(.-)%s*$')
end

-- RFC 2396, section 1.6, 2.2, 2.3 and 2.4.1.
local smartEscape = function(str)
	local pathOffset = str:match("//[^/]+/()")

	-- No path means nothing to escape.<commit_msg>title: Switch to the util library.<commit_after>local util = require'util'
local simplehttp = util.simplehttp
local trim = util.trim
local iconv = require"iconv"
local uri = require"handler.uri"
local x0 = require'x0'
local html2unicode = require'html'
local nixio = require'nixio'

local uri_parse = uri.parse
local DL_LIMIT = 2^17

local patterns = {
	-- X://Y url
	"^(https?://%S+)",
    "^<(https?://%S+)>",
	"%f[%S](https?://%S+)",
	-- www.X.Y url
	"^(www%.[%w_-%%]+%.%S+)",
	"%f[%S](www%.[%w_-%%]+%.%S+)",
}

local translateCharset = {
	utf8 = 'utf-8',
	['x-sjis'] = 'sjis',
	['ks_c_5601-1987'] = 'euc-kr',
	['ksc_5601'] = 'euc-kr',
}

-- RFC 2396, section 1.6, 2.2, 2.3 and 2.4.1.
local smartEscape = function(str)
	local pathOffset = str:match("//[^/]+/()")

	-- No path means nothing to escape.
{'fraction_non_alphanumeric': 0.2503113325031133, 'fraction_numerical': 0.0410958904109589, 'mean_word_length': 3.8549848942598186, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5863573', 'n_tokens_mistral': 740, 'n_tokens_neox': 697, 'n_words': 146}