<commit_before>#pragma once

#include "math/tune.hpp"
#include "accumulator.h"
#include "trainer_result.h"

namespace nano
{
        ///
        /// \brief generic training loop given a suitable optimization operator.
        ///
        template
        <
                typename toperator      ///< (value_accumulator, gradient_accumulator, starting_point)
        >
        trainer_result_t trainer_loop(
                const model_t& model, const size_t nthreads,
                const loss_t& loss, const criterion_t& criterion,
                const toperator& trainer)
        {
                vector_t x0;
                model.save_params(x0);

                // setup accumulators
                accumulator_t lacc(model, loss, criterion, criterion_t::type::value);
                accumulator_t gacc(model, loss, criterion, criterion_t::type::vgrad);

                lacc.set_threads(nthreads);
                gacc.set_threads(nthreads);

                // tune the regularization factor (if needed)
                const auto op = [&] (const scalar_t lambda)
                {
                        lacc.set_lambda(lambda);
                        gacc.set_lambda(lambda);
                        return trainer(lacc, gacc, x0);
                };

                if (lacc.can_regularize())
                {
                        const auto space = nano::make_log10_space(scalar_t(-6.0), scalar_t(+0.0), scalar_t(0.5));
                        return nano::tune(op, space).optimum();
                }
                else
                {
                        return op(0.0);
                }
        }
}<commit_msg>tweak tuning of the regularization parameter<commit_after>#pragma once

#include "math/tune.hpp"
#include "accumulator.h"
#include "trainer_result.h"

namespace nano
{
        ///
        /// \brief generic training loop given a suitable optimization operator.
        ///
        template
        <
                typename toperator      ///< (value_accumulator, gradient_accumulator, starting_point)
        >
        trainer_result_t trainer_loop(
                const model_t& model, const size_t nthreads,
                const loss_t& loss, const criterion_t& criterion,
                const toperator& trainer)
        {
                vector_t x0;
                model.save_params(x0);

                // setup accumulators
                accumulator_t lacc(model, loss, criterion, criterion_t::type::value);
                accumulator_t gacc(model, loss, criterion, criterion_t::type::vgrad);

                lacc.set_threads(nthreads);
                gacc.set_threads(nthreads);

                // tune the regularization factor (if needed)
                const auto op = [&] (const scalar_t lambda)
                {
                        lacc.set_lambda(lambda);
                        gacc.set_lambda(lambda);
                        return trainer(lacc, gacc, x0);
                };

                if (lacc.can_regularize())
                {
                        const auto space = nano::make_log10_space(scalar_t(-5.0), scalar_t(+0.0), scalar_t(0.5), 5);
                        return nano::tune(op, space).optimum();
                }
                else
                {
                        return op(0.0);
                }
        }
}
{'fraction_non_alphanumeric': 0.09279279279279279, 'fraction_numerical': 0.008108108108108109, 'mean_word_length': 1.3408292340126493, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3789130', 'n_tokens_mistral': 954, 'n_tokens_neox': 843, 'n_words': 188}