<commit_before>#include "minibatch.h"
#include "../timer.h"
#include "../logger.h"
#include "../sampler.h"
#include "../minimize.h"
#include "../accumulator.h"
#include "../log_search.hpp"
#include "../thread/thread.h"

namespace ncv
{
        namespace
        {
                void setup_minibatch(sampler_t& orig_tsampler, size_t tsize, trainer_data_t& data)
                {
                        // FIXED random subset of training samples
                        orig_tsampler.setup(sampler_t::stype::uniform, tsize);
                        data.m_tsampler = sampler_t(orig_tsampler.get());
                }

                void reset_minibatch(sampler_t& orig_tsampler, trainer_data_t& data)
                {
                        // all available training samples
                        orig_tsampler.setup(sampler_t::stype::batch);
                        data.m_tsampler = orig_tsampler;
                }

                static size_t make_epoch_size(const trainer_data_t& data, size_t batch)
                {
                        return (data.m_tsampler.size() + batch - 1) / batch;
                }

                template
                <
                        typename toperator
                >
                void train(trainer_data_t& data, size_t epoch_size, size_t batch, const toperator& op)
                {
                        sampler_t orig_tsampler = data.m_tsampler;

                        for (size_t i = 0; i < epoch_size; i ++)
                        {
                                setup_minibatch(orig_tsampler, batch, data);
                                op();
                        }

                        reset_minibatch(orig_tsampler, data);
                }

                static trainer_result_t train(
                        trainer_data_t& data,
                        batch_optimizer optimizer,
                        size_t epochs, size_t epoch_size, size_t batch, size_t iterations, scalar_t epsilon,
                        scalar_t lambda, bool verbose)
                {
                        const ncv::timer_t timer;

                        trainer_result_t result;

                        data.set_lambda(lambda);

                        // construct the optimization problem
                        auto fn_size = ncv::make_opsize(data);
                        auto fn_fval = ncv::make_opfval(data);
                        auto fn_grad = ncv::make_opgrad(data);

                        auto fn_wlog = verbose ? ncv::make_opwlog() : nullptr;
                        auto fn_elog = verbose ? ncv::make_opelog() : nullptr;
                        auto fn_ulog = nullptr;

                        // optimize the model
                        vector_t x = data.m_x0;

                        for (size_t epoch = 1; epoch <= epochs; epoch ++)
                        {
                                train(data, epoch_size, batch, [&] ()
                                {
                                        const opt_state_t state = ncv::minimize(
                                                fn_size, fn_fval, fn_grad, fn_wlog, fn_elog, fn_ulog,
                                                x, optimizer, iterations, epsilon);

                                        x = state.x;
                                });

                                // training samples: loss value
                                data.m_lacc.set_params(x);
                                data.m_lacc.update(data.m_task, data.m_tsampler.all(), data.m_loss);
                                const scalar_t tvalue = data.m_lacc.value();
                                const scalar_t terror_avg = data.m_lacc.avg_error();
                                const scalar_t terror_var = data.m_lacc.var_error();

                                // validation samples: loss value
                                data.m_lacc.set_params(x);
                                data.m_lacc.update(data.m_task, data.m_vsampler.get(), data.m_loss);
                                const scalar_t vvalue = data.m_lacc.value();
                                const scalar_t verror_avg = data.m_lacc.avg_error();
                                const scalar_t verror_var = data.m_lacc.var_error();

                                // update the optimum state
                                result.update(x, tvalue, terror_avg, terror_var, vvalue, verror_avg, verror_var,
                                              epoch, scalars_t({ static_cast<scalar_t>(batch),
                                                                 static_cast<scalar_t>(iterations),
                                                                 lambda }));

                                if (verbose)
                                log_info()
                                        << "[train = " << tvalue << "/" << terror_avg
                                        << ", valid = " << vvalue << "/" << verror_avg
                                        << ", xnorm = " << x.lpNorm<Eigen::Infinity>()
                                        << ", epoch = " << epoch << "/" << epochs
                                        << ", batch = " << batch
                                        << ", iters = " << iterations
                                        << ", lambda = " << data.m_lacc.lambda()
                                        << "] done in " << timer.elapsed() << ".";
                        }

                        return result;
                }
        }

        trainer_result_t minibatch_train(
                const model_t& model,
                const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& criterion,
                batch_optimizer optimizer, size_t epochs, scalar_t epsilon, bool verbose)
        {
                vector_t x0;
                model.save_params(x0);

                // operator to train for a given regularization factor
                const auto op = [&] (scalar_t lambda)
                {
                        accumulator_t lacc(model, nthreads, criterion, criterion_t::type::value, lambda);
                        accumulator_t gacc(model, nthreads, criterion, criterion_t::type::vgrad, lambda);

                        trainer_data_t data(task, tsampler, vsampler, loss, x0, lacc, gacc);

                        const size_t min_batch = 16 * ncv::n_threads();
                        const size_t max_batch = 16 * min_batch;

                        const indices_t batch_iterations = { 4, 8 };

                        trainer_result_t opt_result;
                        size_t opt_batch = min_batch;
                        size_t opt_iterations = 4;

                        // tune the batch size and the number of optimization iterations per batch
                        for (size_t batch = min_batch; batch <= max_batch; batch *= 2)
                        {
                                for (size_t iterations : batch_iterations)
                                {
                                        const ncv::timer_t timer;

                                        const size_t epoch_size = make_epoch_size(data, batch);
                                        const size_t epochs = 1;

                                        const trainer_result_t result =
                                                train(data, optimizer, epochs, epoch_size, batch,
                                                              iterations, epsilon, lambda, false);

                                        const trainer_state_t& state = result.m_opt_state;

                                        if (verbose)
                                        log_info()
                                                << "[tuning: train = " << state.m_tvalue << "/" << state.m_terror_avg
                                                << ", valid = " << state.m_vvalue << "/" << state.m_verror_avg
                                                << ", batch = " << batch
                                                << ", iters = " << iterations
                                                << ", lambda = " << data.m_lacc.lambda()
                                                << "] done in " << timer.elapsed() << ".";

                                        if (result < opt_result)
                                        {
                                                opt_result = result;
                                                opt_batch = batch;
                                                opt_iterations = iterations;
                                        }
                                }
                        }

                        // train the model using the tuned parameters
                        const size_t epoch_size = make_epoch_size(data, opt_batch);

                        return train(data, optimizer, epochs, epoch_size, opt_batch,
                                             opt_iterations, epsilon, lambda, verbose);
                };

                // tune the regularization factor (if needed)
                if (accumulator_t::can_regularize(criterion))
                {
                       return log10_min_search(op, -6.0, +0.0, 0.2, 4).first;
                }
                else
                {
                        return op(0.0);
                }
        }
}
	<commit_msg>faster tuning of the regularization term (use a single epoch)<commit_after>#include "minibatch.h"
#include "../timer.h"
#include "../logger.h"
#include "../sampler.h"
#include "../minimize.h"
#include "../accumulator.h"
#include "../log_search.hpp"
#include "../thread/thread.h"
#include <tuple>

namespace ncv
{
        namespace
        {
                void setup_minibatch(sampler_t& orig_tsampler, size_t tsize, trainer_data_t& data)
                {
                        // FIXED random subset of training samples
                        orig_tsampler.setup(sampler_t::stype::uniform, tsize);
                        data.m_tsampler = sampler_t(orig_tsampler.get());
                }

                void reset_minibatch(sampler_t& orig_tsampler, trainer_data_t& data)
                {
                        // all available training samples
                        orig_tsampler.setup(sampler_t::stype::batch);
                        data.m_tsampler = orig_tsampler;
                }

                size_t make_epoch_size(const trainer_data_t& data, size_t batch)
                {
                        return (data.m_tsampler.size() + batch - 1) / batch;
                }

                template
                <
                        typename toperator
                >
                void train(trainer_data_t& data, size_t epoch_size, size_t batch, const toperator& op)
                {
                        sampler_t orig_tsampler = data.m_tsampler;

                        for (size_t i = 0; i < epoch_size; i ++)
                        {
                                setup_minibatch(orig_tsampler, batch, data);
                                op();
                        }

                        reset_minibatch(orig_tsampler, data);
                }

                trainer_result_t train(
                        trainer_data_t& data,
                        batch_optimizer optimizer,
                        size_t epochs, size_t batch, size_t iterations, scalar_t epsilon,
                        bool verbose)
                {
                        const ncv::timer_t timer;

                        trainer_result_t result;

                        const size_t epoch_size = make_epoch_size(data, batch);

                        // construct the optimization problem
                        auto fn_size = ncv::make_opsize(data);
                        auto fn_fval = ncv::make_opfval(data);
                        auto fn_grad = ncv::make_opgrad(data);

                        auto fn_wlog = verbose ? ncv::make_opwlog() : nullptr;
                        auto fn_elog = verbose ? ncv::make_opelog() : nullptr;
                        auto fn_ulog = nullptr;

                        // optimize the model
                        vector_t x = data.m_x0;

                        for (size_t epoch = 1; epoch <= epochs; epoch ++)
                        {
                                train(data, epoch_size, batch, [&] ()
                                {
                                        const opt_state_t state = ncv::minimize(
                                                fn_size, fn_fval, fn_grad, fn_wlog, fn_elog, fn_ulog,
                                                x, optimizer, iterations, epsilon);

                                        x = state.x;
                                });

                                // training samples: loss value
                                data.m_lacc.set_params(x);
                                data.m_lacc.update(data.m_task, data.m_tsampler.all(), data.m_loss);
                                const scalar_t tvalue = data.m_lacc.value();
                                const scalar_t terror_avg = data.m_lacc.avg_error();
                                const scalar_t terror_var = data.m_lacc.var_error();

                                // validation samples: loss value
                                data.m_lacc.set_params(x);
                                data.m_lacc.update(data.m_task, data.m_vsampler.get(), data.m_loss);
                                const scalar_t vvalue = data.m_lacc.value();
                                const scalar_t verror_avg = data.m_lacc.avg_error();
                                const scalar_t verror_var = data.m_lacc.var_error();

                                // update the optimum state
                                result.update(x, tvalue, terror_avg, terror_var, vvalue, verror_avg, verror_var,
                                              epoch, scalars_t({ static_cast<scalar_t>(batch),
                                                                 static_cast<scalar_t>(iterations),
                                                                 data.lambda() }));

                                if (verbose)
                                log_info()
                                        << "[train = " << tvalue << "/" << terror_avg
                                        << ", valid = " << vvalue << "/" << verror_avg
                                        << ", xnorm = " << x.lpNorm<Eigen::Infinity>()
                                        << ", epoch = " << epoch << "/" << epochs
                                        << ", batch = " << batch
                                        << ", iters = " << iterations
                                        << ", lambda = " << data.lambda()
                                        << "] done in " << timer.elapsed() << ".";
                        }

                        return result;
                }

                std::tuple<trainer_result_t, size_t, size_t> tune_minibatch(
                        trainer_data_t& data, batch_optimizer optimizer, scalar_t epsilon, bool verbose)
                {
                        const size_t min_batch = 16 * ncv::n_threads();
                        const size_t max_batch = 16 * min_batch;

                        const indices_t batch_iterations = { 4, 8 };

                        trainer_result_t opt_result;
                        size_t opt_batch = min_batch;
                        size_t opt_iterations = 4;

                        // tune the batch size and the number of optimization iterations per batch
                        for (size_t batch = min_batch; batch <= max_batch; batch *= 2)
                        {
                                for (size_t iterations : batch_iterations)
                                {
                                        const ncv::timer_t timer;

                                        const size_t epochs = 1;
                                        const trainer_result_t result =
                                                train(data, optimizer, epochs, batch, iterations, epsilon, false);

                                        const trainer_state_t& state = result.m_opt_state;

                                        if (verbose)
                                        log_info()
                                                << "[tuning: train = " << state.m_tvalue << "/" << state.m_terror_avg
                                                << ", valid = " << state.m_vvalue << "/" << state.m_verror_avg
                                                << ", batch = " << batch
                                                << ", iters = " << iterations
                                                << ", lambda = " << data.lambda()
                                                << "] done in " << timer.elapsed() << ".";

                                        if (result < opt_result)
                                        {
                                                opt_result = result;
                                                opt_batch = batch;
                                                opt_iterations = iterations;
                                        }
                                }
                        }

                        // OK
                        return std::make_tuple(opt_result, opt_batch, opt_iterations);
                }

                std::tuple<trainer_result_t, size_t, size_t, scalar_t> tune_lambda(
                        trainer_data_t& data, batch_optimizer optimizer, scalar_t epsilon, bool verbose)
                {
                        const auto op = [&] (scalar_t lambda)
                        {
                                data.set_lambda(lambda);

                                return tune_minibatch(data, optimizer, epsilon, verbose);
                        };

                        const auto ret = log10_min_search(op, -6.0, +0.0, 0.5, 4);

                        return std::make_tuple(std::get<0>(ret.first),
                                               std::get<1>(ret.first),
                                               std::get<2>(ret.first),
                                               ret.second);
                }
        }

        trainer_result_t minibatch_train(
                const model_t& model,
                const task_t& task, const sampler_t& tsampler, const sampler_t& vsampler, size_t nthreads,
                const loss_t& loss, const string_t& criterion,
                batch_optimizer optimizer, size_t epochs, scalar_t epsilon, bool verbose)
        {
                vector_t x0;
                model.save_params(x0);

                // setup acumulators
                accumulator_t lacc(model, nthreads, criterion, criterion_t::type::value);
                accumulator_t gacc(model, nthreads, criterion, criterion_t::type::vgrad);

                trainer_data_t data(task, tsampler, vsampler, loss, x0, lacc, gacc);

                // tune the regularization factor (if needed)
                if (accumulator_t::can_regularize(criterion))
                {
                        const auto ret = tune_lambda(data, optimizer, epsilon, verbose);

                        const size_t opt_batch = std::get<1>(ret);
                        const size_t opt_iterations = std::get<2>(ret);
                        const size_t opt_lambda = std::get<3>(ret);

                        data.set_lambda(opt_lambda);

                        return train(data, optimizer, epochs, opt_batch, opt_iterations, epsilon, verbose);
                }

                else
                {
                        const auto ret = tune_minibatch(data, optimizer, epsilon, verbose);

                        const size_t opt_batch = std::get<1>(ret);
                        const size_t opt_iterations = std::get<2>(ret);

                        return train(data, optimizer, epochs, opt_batch, opt_iterations, epsilon, verbose);
                }
        }
}
	
{'fraction_non_alphanumeric': 0.08104497550838652, 'fraction_numerical': 0.0029686804215526197, 'mean_word_length': 0.9001598194979787, 'pattern_counts': {'":': 0, '<': 190, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 141, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3495899', 'n_tokens_mistral': 4841, 'n_tokens_neox': 4424, 'n_words': 990}