<commit_before>local json = require "lunajson"

--
-- This file provides a convenience high-level function called `run`, for training or evaluating a
-- model on a given task. The user can invoke this function to get started with a minimal amount of
-- boilerplate code. Several optional arguments allow a great deal of flexibility and customization
-- of the training process.
--

-- We do careful verification here to minimize our risk of running into easily-preventable problems
-- much later down the line.
local function validate_args(args)
	assert(args)
	assert(type(args) == "table")
	assert(args.model)
	assert(args.driver)
	assert(args.perf_metrics)
	assert(type(args.perf_metrics) == "table")
	assert(args.model_dir)
	assert(type(args.model_dir) == "string")

	-- Convert `perf_metrics` to a key-value table if it is an array.
	if args.perf_metrics[1] then
		local tmp = {}
		for _, v in pairs(args.perf_metrics) do
			-- Here, `true` is just used as a dummy value.
			tmp[v] = true
		end
		args.perf_metrics = tmp
	end
end

--
-- The arguments to this function must be provided in a table.
--
-- Parameters:
-- * model: An implementation of the Model interface.
-- * driver: Class responsible for performing the training and testing epochs.
-- * perf_metrics: A table of strings indicating which performance metrics to compute. If
--   `perf_metrics[m] == "not important"` for some metric `m`, then the model will not necessarily
--   be saved each time `m` is improved. By default, the model is saved if **any** performance
--   metric improves after a training epoch. Valid values: any metric in the table
--   `lantern.performance_metrics`.
-- * model_dir: Directory in which to serialize models and other state information.
-- * history (optional): Used to resume progress from a previous training trajectory.
-- * test_epoch_ratio (optional): The number of training epochs to perform before running a test
--   epoch. Default: 1.
-- * optimizer (optional): Optimizer used to update the model. The default optimizer is
--   `lantern.optimizers.adadelta_lm` with default parameters.
-- * stop_crit (optional): Criterion used to determine when to stop training.  Default: stop if
--   there is no improvement in **any** performance metric after 10 epochs.
-- * logger (optional): Used to log real-time events. Default: `lantern.stdout_logger`.
--
function lantern.run(args)
	validate_args(args)

	-- Make aliases to the required arguments.
	local model        = args.model
	local model_dir    = args.model_dir
	local driver       = args.driver
	local perf_metrics = args.perf_metrics

	-- Infer the values of the optional arguments.
	local optim            = args.optimizer or lantern.optimizers.adadelta_lm(model)
	local stop_crit        = args.stop_crit or lantern.criterion.max_epochs_per_improvement(10)
	local test_epoch_ratio = args.test_epoch_ratio or 1
	local logger           = args.logger or lantern.stdout_logger()
	local hist             = args.history or {{epoch = 1}}

	assert(#hist >= 1)
	logger:update("/history", hist)
<commit_msg>Fixed formatting.<commit_after>local json = require "lunajson"

--
-- This file provides a convenience high-level function called `run`, for training or evaluating a
-- model on a given task. The user can invoke this function to get started with a minimal amount of
-- boilerplate code. Several optional arguments allow a great deal of flexibility and customization
-- of the training process.
--

-- We do careful verification here to minimize our risk of running into easily-preventable problems
-- much later down the line.
local function validate_args(args)
	assert(args)
	assert(type(args) == "table")
	assert(args.model)
	assert(args.driver)
	assert(args.perf_metrics)
	assert(type(args.perf_metrics) == "table")
	assert(args.model_dir)
	assert(type(args.model_dir) == "string")

	-- Convert `perf_metrics` to a key-value table if it is an array.
	if args.perf_metrics[1] then
		local tmp = {}
		for _, v in pairs(args.perf_metrics) do
			-- Here, `true` is just used as a dummy value.
			tmp[v] = true
		end
		args.perf_metrics = tmp
	end
end

--
-- The arguments to this function must be provided in a table.
--
-- Parameters:
-- * model: An implementation of the Model interface.
-- * driver: Class responsible for performing the training and testing epochs.
-- * perf_metrics: A table of strings indicating which performance metrics to compute. If
--   `perf_metrics[m] == "not important"` for some metric `m`, then the model will not necessarily
--   be saved each time `m` is improved. By default, the model is saved if **any** performance
--   metric improves after a training epoch. Valid values: any metric in the table
--   `lantern.performance_metrics`.
-- * model_dir: Directory in which to serialize models and other state information.
-- * history (optional): Used to resume progress from a previous training trajectory.
-- * test_epoch_ratio (optional): The number of training epochs to perform before running a test
--   epoch. Default: 1.
-- * optimizer (optional): Optimizer used to update the model. The default optimizer is
--   `lantern.optimizers.adadelta_lm` with default parameters.
-- * stop_crit (optional): Criterion used to determine when to stop training. Default: stop if there
--   is no improvement in **any** performance metric after 10 epochs.
-- * logger (optional): Used to log real-time events. Default: `lantern.stdout_logger`.
--
function lantern.run(args)
	validate_args(args)

	-- Make aliases to the required arguments.
	local model        = args.model
	local model_dir    = args.model_dir
	local driver       = args.driver
	local perf_metrics = args.perf_metrics

	-- Infer the values of the optional arguments.
	local optim            = args.optimizer or lantern.optimizers.adadelta_lm(model)
	local stop_crit        = args.stop_crit or lantern.criterion.max_epochs_per_improvement(10)
	local test_epoch_ratio = args.test_epoch_ratio or 1
	local logger           = args.logger or lantern.stdout_logger()
	local hist             = args.history or {{epoch = 1}}

	assert(#hist >= 1)
	logger:update("/history", hist)

{'fraction_non_alphanumeric': 0.09224489795918367, 'fraction_numerical': 0.0029387755102040815, 'mean_word_length': 3.655015197568389, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5274491', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1690, 'n_words': 707}