<commit_before>local strings = {}

if #arg < 2 then
  print("Usage: Babel.lua varname sourcefile ...\n"..
        "  varname - the variable name of the global table holding the translations\n"..
        "  sourcefile ... - the source files to scavenge for localized strings\n")
end

-- The first arg is the global variable to use for the global table
local varname = arg[1]

-- Parse all the lua files given in the command line and extract
-- strings for localization.
for i = 2,#arg do
  local file = io.open(arg[i], "r")
  assert(file, "Could not open "..arg[i])
  print("Reading "..arg[i])
  local text = file:read("*all")

  for match in string.gmatch(text, "L%[\"(.-)\"%]") do
    table.insert(strings, match)
  end
end

local languages = {
  "enUS",
  "frFR",
  "deDE",
}<commit_msg>Update with all knows WoW localized clients.<commit_after>local strings = {}

if #arg < 2 then
  print("Usage: Babel.lua varname sourcefile ...\n"..
        "  varname - the variable name of the global table holding the translations\n"..
        "  sourcefile ... - the source files to scavenge for localized strings\n")
end

-- The first arg is the global variable to use for the global table
local varname = arg[1]

-- Parse all the lua files given in the command line and extract
-- strings for localization.
for i = 2,#arg do
  local file = io.open(arg[i], "r")
  assert(file, "Could not open "..arg[i])
  print("Reading "..arg[i])
  local text = file:read("*all")

  for match in string.gmatch(text, "L%[\"(.-)\"%]") do
    table.insert(strings, match)
  end
end

local languages = {
  "enUS",
  "frFR",
  "deDE",
  "zhCN",
  "zhTW",
  "koKR",
  "esES",
}
{'fraction_non_alphanumeric': 0.14493632504548212, 'fraction_numerical': 0.0036385688295936932, 'mean_word_length': 3.0441176470588234, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4847281', 'n_tokens_mistral': 567, 'n_tokens_neox': 542, 'n_words': 202}