<commit_before>#include "math/stats.h"
#include "math/numeric.h"
#include "math/epsilon.h"
#include "trainer_state.h"
#include "text/algorithm.h"
#include "text/to_string.h"
#include <fstream>

namespace nano
{
        trainer_measurement_t::trainer_measurement_t() : trainer_measurement_t(
                std::numeric_limits<scalar_t>::max(),
                std::numeric_limits<scalar_t>::max())
        {
        }

        trainer_measurement_t::trainer_measurement_t(
                const scalar_t value, const scalar_t error) :
                m_value(value), m_error(error)
        {
        }

        trainer_measurement_t::operator bool() const
        {
                return  std::isfinite(m_value) &&
                        std::isfinite(m_value);
        }

        bool operator<(const trainer_measurement_t& one, const trainer_measurement_t& two)
        {
                const auto v1 = (one) ? one.m_value : std::numeric_limits<scalar_t>::max();
                const auto v2 = (two) ? two.m_value : std::numeric_limits<scalar_t>::max();
                return v1 < v2;
        }

        trainer_state_t::trainer_state_t() :
                m_milis(0),
                m_epoch(0),
                m_xnorm(0),
                m_gnorm(0)
        {
        }

        trainer_state_t::trainer_state_t(
                const milliseconds_t milis,
                const size_t epoch,
                const scalar_t xnorm,
                const scalar_t gnorm,<commit_msg>fix check for valid training measurement<commit_after>#include "math/stats.h"
#include "math/numeric.h"
#include "math/epsilon.h"
#include "trainer_state.h"
#include "text/algorithm.h"
#include "text/to_string.h"
#include <fstream>

namespace nano
{
        trainer_measurement_t::trainer_measurement_t() : trainer_measurement_t(
                std::numeric_limits<scalar_t>::max(),
                std::numeric_limits<scalar_t>::max())
        {
        }

        trainer_measurement_t::trainer_measurement_t(
                const scalar_t value, const scalar_t error) :
                m_value(value), m_error(error)
        {
        }

        trainer_measurement_t::operator bool() const
        {
                return  std::isfinite(m_value) &&
                        std::isfinite(m_error);
        }

        bool operator<(const trainer_measurement_t& one, const trainer_measurement_t& two)
        {
                const auto v1 = (one) ? one.m_value : std::numeric_limits<scalar_t>::max();
                const auto v2 = (two) ? two.m_value : std::numeric_limits<scalar_t>::max();
                return v1 < v2;
        }

        trainer_state_t::trainer_state_t() :
                m_milis(0),
                m_epoch(0),
                m_xnorm(0),
                m_gnorm(0)
        {
        }

        trainer_state_t::trainer_state_t(
                const milliseconds_t milis,
                const size_t epoch,
                const scalar_t xnorm,
                const scalar_t gnorm,
{'fraction_non_alphanumeric': 0.10442301296973729, 'fraction_numerical': 0.005320917858330562, 'mean_word_length': 1.853889943074004, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1963770', 'n_tokens_mistral': 942, 'n_tokens_neox': 905, 'n_words': 154}