<commit_before>-------------------------------------------------------------------------------
-- General utilities.
-- @author <NAME>
-- @copyright <NAME>
-------------------------------------------------------------------------------

--- Split a string using '@sep' separator
-- @param sep char to split string
-- @usage 'a string, with some words':split(',')
-- @usage 'some:data:to:split':split()
-- @return table with all split elements using the separator char
-------------------------------------------------------------------------------
function string:split(sep)
  local sep, fields = sep or ":", {}
  local pattern = string.format("([^%s]+)", sep)
  self:gsub(pattern, function(c) fields[#fields+1] = c end)
  return fields
end

local utils = { }
-------------------------------------------------------------------------------
--- Cut off trailing spaces from string's beginning
-- @param string string to be replaced
-- @usage deleteBeginSpace('a string') => 'a string'
-- @usage deleteBeginSpace('   a string') => 'a string'
-- @usage deleteBeginSpace('   a string    ') => 'a string    '
-- @return a string without any space at the beginning
-------------------------------------------------------------------------------
function utils.deleteBeginSpace(string)<commit_msg>missing require() for lfs dependency<commit_after>-------------------------------------------------------------------------------
-- General utilities.
-- @author <NAME>
-- @copyright Olivier DOSSMANN
-------------------------------------------------------------------------------

local lfs = require "lfs"

--- Split a string using '@sep' separator
-- @param sep char to split string
-- @usage 'a string, with some words':split(',')
-- @usage 'some:data:to:split':split()
-- @return table with all split elements using the separator char
-------------------------------------------------------------------------------
function string:split(sep)
  local sep, fields = sep or ":", {}
  local pattern = string.format("([^%s]+)", sep)
  self:gsub(pattern, function(c) fields[#fields+1] = c end)
  return fields
end

local utils = { }
-------------------------------------------------------------------------------
--- Cut off trailing spaces from string's beginning
-- @param string string to be replaced
-- @usage deleteBeginSpace('a string') => 'a string'
-- @usage deleteBeginSpace('   a string') => 'a string'
-- @usage deleteBeginSpace('   a string    ') => 'a string    '
-- @return a string without any space at the beginning
-------------------------------------------------------------------------------
function utils.deleteBeginSpace(string)
{'fraction_non_alphanumeric': 0.4034090909090909, 'fraction_numerical': 0.0007575757575757576, 'mean_word_length': 1.265008576329331, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2429412', 'n_tokens_mistral': 566, 'n_tokens_neox': 556, 'n_words': 206}