<commit_before>#include "stochastic_trainer.h"
#include "core/timer.h"
#include "core/text.h"
#include "core/math/clamp.hpp"
#include "core/thread/thread.h"
#include "core/logger.h"

namespace ncv
{
        /////////////////////////////////////////////////////////////////////////////////////////

        stochastic_trainer_t::stochastic_trainer_t(const string_t& params)
                :       m_iterations(text::from_params<size_t>(params, "iters", 1024)),
                        m_depth(text::from_params<size_t>(params, "depth", 4))
        {
                m_iterations = math::clamp(m_iterations, 100, 10000);
                m_depth = math::clamp(m_depth, 2, 16);
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        scalar_t stochastic_trainer_t::sgd(
                const task_t& task, const samples_t& samples, const loss_t& loss, model_t& model, vector_t& x,
                scalar_t gamma, scalar_t lambda, size_t iterations, size_t evalsize) const
        {                
                vector_t avg_x = x;

                // (A=average)SGD steps
                random_t<size_t> die(0, samples.size() - 1);
                for (size_t iteration = 0; iteration < iterations; iteration ++)
                {
                        const sample_t& sample = samples[die()];

                        model.zero_grad();
                        model.load_params(x);

                        const scalar_t f = ncv::lvgrad(task, sample, loss, model);
                        const scalar_t d = learning_rate(gamma, lambda, iteration);
                        const vector_t g = model.grad();

                        if (    std::isinf(f) || std::isinf(g.minCoeff()) || std::isinf(g.maxCoeff()) ||
                                std::isnan(f) || std::isnan(g.minCoeff()) || std::isnan(g.maxCoeff()))
                        {
                                return std::numeric_limits<scalar_t>::max();
                        }

                        x -= d * g;
                        avg_x += x;
                }

                model.load_params(avg_x / (1.0 + iterations));

                // evaluate model
                samples_t esamples;
                for (size_t i = 0; i < evalsize; i ++)
                {
                        const sample_t& sample = samples[die()];

                        esamples.emplace_back(sample);
                }

                return ncv::lvalue_st(task, esamples, loss, model);
        }

        /////////////////////////////////////////////////////////////////////////////////////////
<commit_msg>stochastic trainer: better random sampling - no convergence speed improvement though<commit_after>#include "stochastic_trainer.h"
#include "core/timer.h"
#include "core/text.h"
#include "core/math/clamp.hpp"
#include "core/thread/thread.h"
#include "core/logger.h"

namespace ncv
{
        /////////////////////////////////////////////////////////////////////////////////////////

        stochastic_trainer_t::stochastic_trainer_t(const string_t& params)
                :       m_iterations(text::from_params<size_t>(params, "iters", 1024)),
                        m_depth(text::from_params<size_t>(params, "depth", 4))
        {
                m_iterations = math::clamp(m_iterations, 100, 10000);
                m_depth = math::clamp(m_depth, 2, 16);
        }

        /////////////////////////////////////////////////////////////////////////////////////////

        scalar_t stochastic_trainer_t::sgd(
                const task_t& task, const samples_t& samples, const loss_t& loss, model_t& model, vector_t& x,
                scalar_t gamma, scalar_t lambda, size_t iterations, size_t evalsize) const
        {                
                vector_t avg_x = x;

                // (A=average)SGD steps
                for (size_t iteration = 0; iteration < iterations; iteration ++)
                {                        
                        random_t<size_t> die(0, samples.size() - 1);
                        const sample_t& sample = samples[die()];

                        model.zero_grad();
                        model.load_params(x);

                        const scalar_t f = ncv::lvgrad(task, sample, loss, model);
                        const scalar_t d = learning_rate(gamma, lambda, iteration);
                        const vector_t g = model.grad();

                        if (    std::isinf(f) || std::isinf(g.minCoeff()) || std::isinf(g.maxCoeff()) ||
                                std::isnan(f) || std::isnan(g.minCoeff()) || std::isnan(g.maxCoeff()))
                        {
                                return std::numeric_limits<scalar_t>::max();
                        }

                        x -= d * g;
                        avg_x += x;
                }

                model.load_params(avg_x / (1.0 + iterations));

                // evaluate model
                random_t<size_t> die(0, samples.size() - 1);

                samples_t esamples(evalsize);
                for (sample_t& sample : esamples)
                {
                        sample = samples[die()];
                }

                return ncv::lvalue_st(task, esamples, loss, model);
        }

        /////////////////////////////////////////////////////////////////////////////////////////

{'fraction_non_alphanumeric': 0.20137597619933062, 'fraction_numerical': 0.0083674228337672, 'mean_word_length': 1.086501163692785, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1540434', 'n_tokens_mistral': 1497, 'n_tokens_neox': 1340, 'n_words': 271}