<commit_before>


local function loadfilein(file, env) return loadin and loadin(env, io.open(file):read("*a")) or setfenv(loadfile(file), env); end
config = {};
local config_env = setmetatable({}, { __index = function(t, k) return function(tbl) config[k] = tbl; end; end });
loadfilein("config.lua", config_env)();

package.path = "../../?.lua;"..package.path
package.cpath = "../../?.dll;"..package.cpath


assert(config.input, "no input specified")
assert(config.output, "no output specified")
local itype = assert(config.input.type, "no input.type specified");
local otype = assert(config.output.type, "no output.type specified");
local reader = require(itype).reader(config.input);
local writer = require(otype).writer(config.output);

local json = require "util.json";

for x in reader do
	--print(json.encode(x))
	writer(x);
end
writer(nil); -- close
<commit_msg>tools/migration/main.lua: Add command-line parsing, including --config=CONFIG_FILE, and the ability to specify to/from stores to migrate<commit_after>-- Command-line parsing
local options = {};
local handled_opts = 0;
for i = 1, #arg do
	if arg[i]:sub(1,2) == "--" then
		local opt, val = arg[i]:match("([%w-]+)=?(.*)");
		if opt then
			options[(opt:sub(3):gsub("%-", "_"))] = #val > 0 and val or true;
		end
		handled_opts = i;
	else
		break;
	end
end
table.remove(arg, handled_opts);

-- Load config file
local function loadfilein(file, env) return loadin and loadin(env, io.open(file):read("*a")) or setfenv(loadfile(file), env); end
config = {};
local config_env = setmetatable({}, { __index = function(t, k) return function(tbl) config[k] = tbl; end; end });
loadfilein(options.config or "config.lua", config_env)();

if not package.loaded["util.json"] then
	package.path = "../../?.lua;"..package.path
	package.cpath = "../../?.dll;"..package.cpath
end

local from_store = arg[1] or "input";
local to_store = arg[2] or "output";

assert(config[from_store], "no input specified")
assert(config[to_store], "no output specified")
local itype = assert(config[from_store].type, "no type specified for "..from_store);
local otype = assert(config[to_store].type, "no type specified for "..to_store);
local reader = require(itype).reader(config[from_store]);
local writer = require(otype).writer(config[to_store]);

local json = require "util.json";

for x in reader do
	--print(json.encode(x))
	writer(x);
end
writer(nil); -- close

{'fraction_non_alphanumeric': 0.1790920449812578, 'fraction_numerical': 0.003331945022907122, 'mean_word_length': 4.960297766749379, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6478512', 'n_tokens_mistral': 815, 'n_tokens_neox': 799, 'n_words': 222}