<commit_before>-- Strips string of leading and trailing spaces and newlines
local function trim(self)
  return string.match(self, '^%s*(.*%S)') or ''
end

-- Splits string into pieces
local function split(str, div)
    assert(type(str) == "string" and type(div) == "string", "invalid arguments")
    assert(not string.find("", div, 1), "delimiter matches empty string!")
    local res = {}
    while true do
        local pos1,pos2 = str:find(div)
        if not pos1 then
            res[#res+1] = str
            break
        end
        res[#res+1],str = str:sub(1,pos1-1),str:sub(pos2+1)
    end
    return res
end

local function hex_to_char(x)
  return string.char(tonumber(x, 16))
end

local function decodeUrl(url)
  url = url:gsub("%%(%x%x)", hex_to_char)
  url = url:gsub("%%%%", "%%")
  return url
end

return { 
    split=split, 
    trim=trim,
    decodeUrl=decodeUrl
    }<commit_msg>added join wrapper for table.concat<commit_after>-- Strips string of leading and trailing spaces and newlines
local function trim(self)
  return string.match(self, '^%s*(.*%S)') or ''
end

local function join(tbl, glue)
    return table.concat(tbl, glue)
end

-- Splits string into pieces
local function split(str, div)
    assert(type(str) == "string" and type(div) == "string", "invalid arguments")
    assert(not string.find("", div, 1), "delimiter matches empty string!")
    local res = {}
    while true do
        local pos1,pos2 = str:find(div)
        if not pos1 then
            res[#res+1] = str
            break
        end
        res[#res+1],str = str:sub(1,pos1-1),str:sub(pos2+1)
    end
    return res
end

local function hex_to_char(x)
  return string.char(tonumber(x, 16))
end

local function decodeUrl(url)
  url = url:gsub("%%(%x%x)", hex_to_char)
  url = url:gsub("%%%%", "%%")
  return url
end

return { 
    split=split, 
    join=join,
    trim=trim,
    decodeUrl=decodeUrl
}
{'fraction_non_alphanumeric': 0.1440588853838065, 'fraction_numerical': 0.013669821240799159, 'mean_word_length': 3.1459694989106755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3451343', 'n_tokens_mistral': 667, 'n_tokens_neox': 649, 'n_words': 183}