<commit_before>#include <memory>

#include <Eigen/Sparse>
#include <Eigen/Core>

#include <SmurffCpp/Utils/counters.h>
#include <SmurffCpp/Utils/RootFile.h>
#include <SmurffCpp/Utils/MatrixUtils.h>
#include <SmurffCpp/result.h>
#include <SmurffCpp/ResultItem.h>

#include <SmurffCpp/Model.h>

#include <SmurffCpp/Predict/PredictSession.h>

namespace smurff
{

PredictSession::PredictSession(std::shared_ptr<RootFile> rf)
    : m_model_rootfile(rf), m_pred_rootfile(0),
      m_has_config(false), m_num_latent(-1),
      m_dims(PVec<>(0)), m_is_init(false)
{
    m_stepfiles = m_model_rootfile->openSampleStepFiles();
}

PredictSession::PredictSession(std::shared_ptr<RootFile> rf, const Config &config)
    : m_model_rootfile(rf), m_pred_rootfile(0),
      m_config(config), m_has_config(true), m_num_latent(-1),
      m_dims(PVec<>(0)), m_is_init(false)
{
    m_stepfiles = m_model_rootfile->openSampleStepFiles();
}
PredictSession::PredictSession(const Config &config)
    : m_pred_rootfile(0), m_config(config), m_has_config(true),
      m_num_latent(-1), m_dims(PVec<>(0)), m_is_init(false)
{
    THROWERROR_ASSERT(config.getRootName().size())
    m_model_rootfile = std::make_shared<RootFile>(config.getRootName());
    m_stepfiles = m_model_rootfile->openSampleStepFiles();
}

void PredictSession::run()
{
    THROWERROR_ASSERT(m_has_config);

    std::vector<std::shared_ptr<Eigen::MatrixXd>> predictions;
    std::shared_ptr<MatrixConfig> side_info;
    int mode;

    if (m_config.getTest())
    {
        init();
        while (step())
            ;

        return;
    }
    else if (m_config.getRowFeatures())
    {
        side_info = m_config.getRowFeatures();
        mode = 0;
    }
    else if (m_config.getColFeatures())
    {
        side_info = m_config.getColFeatures();
        mode = 1;
    }
    else 
    {
        THROWERROR("Need either test, row features or col features")
    }

    if (side_info->isDense())
    {
        auto dense_matrix = matrix_utils::dense_to_eigen(*side_info);
        predictions = predict(mode, dense_matrix);
    }
    else
    {
        auto sparse_matrix = matrix_utils::sparse_to_eigen(*side_info);
        predictions = predict(mode, sparse_matrix);
    }
}

void PredictSession::init()
{
    THROWERROR_ASSERT(m_has_config);
    THROWERROR_ASSERT(m_config.getTest());
    m_result = std::make_shared<Result>(m_config.getTest(), m_config.getNSamples());

    m_pos = m_stepfiles.rbegin();
    m_iter = 0;
    m_is_init = true;

    THROWERROR_ASSERT_MSG(m_config.getSavePrefix() != getModelRoot()->getPrefix(),
                          "Cannot have same prefix for model and predictions - both have " + m_config.getSavePrefix());

    if (m_config.getSaveFreq())
    {
        // create root file
        m_pred_rootfile = std::make_shared<RootFile>(m_config.getSavePrefix(), m_config.getSaveExtension(), true);
        m_pred_rootfile->createCsvStatusFile();
        m_pred_rootfile->flushLast();
    }

    if (m_config.getVerbose())
        info(std::cout, "");<commit_msg>Second step to out-of-matrix predictions in smurff command line verison<commit_after>#include <memory>

#include <Eigen/Sparse>
#include <Eigen/Core>

#include <SmurffCpp/Utils/counters.h>
#include <SmurffCpp/Utils/RootFile.h>
#include <SmurffCpp/Utils/MatrixUtils.h>
#include <SmurffCpp/IO/MatrixIO.h>
#include <SmurffCpp/result.h>
#include <SmurffCpp/ResultItem.h>

#include <SmurffCpp/Model.h>

#include <SmurffCpp/Predict/PredictSession.h>

namespace smurff
{

PredictSession::PredictSession(std::shared_ptr<RootFile> rf)
    : m_model_rootfile(rf), m_pred_rootfile(0),
      m_has_config(false), m_num_latent(-1),
      m_dims(PVec<>(0)), m_is_init(false)
{
    m_stepfiles = m_model_rootfile->openSampleStepFiles();
}

PredictSession::PredictSession(std::shared_ptr<RootFile> rf, const Config &config)
    : m_model_rootfile(rf), m_pred_rootfile(0),
      m_config(config), m_has_config(true), m_num_latent(-1),
      m_dims(PVec<>(0)), m_is_init(false)
{
    m_stepfiles = m_model_rootfile->openSampleStepFiles();
}
PredictSession::PredictSession(const Config &config)
    : m_pred_rootfile(0), m_config(config), m_has_config(true),
      m_num_latent(-1), m_dims(PVec<>(0)), m_is_init(false)
{
    THROWERROR_ASSERT(config.getRootName().size())
    m_model_rootfile = std::make_shared<RootFile>(config.getRootName());
    m_stepfiles = m_model_rootfile->openSampleStepFiles();
}

void PredictSession::run()
{
    THROWERROR_ASSERT(m_has_config);

    std::vector<std::shared_ptr<Eigen::MatrixXd>> predictions;
    std::shared_ptr<MatrixConfig> side_info;
    int mode;

    if (m_config.getTest())
    {
        init();
        while (step())
            ;

        return;
    }
    else if (m_config.getRowFeatures())
    {
        side_info = m_config.getRowFeatures();
        mode = 0;
    }
    else if (m_config.getColFeatures())
    {
        side_info = m_config.getColFeatures();
        mode = 1;
    }
    else 
    {
        THROWERROR("Need either test, row features or col features")
    }

    if (side_info->isDense())
    {
        auto dense_matrix = matrix_utils::dense_to_eigen(*side_info);
        predictions = predict(mode, dense_matrix);
    }
    else
    {
        auto sparse_matrix = matrix_utils::sparse_to_eigen(*side_info);
        predictions = predict(mode, sparse_matrix);
    }

    int i = 0;
    for (auto p : predictions)
    {
        auto filename = m_config.getSavePrefix() + "/predictions-sample-" + std::to_string(i) + m_config.getSaveExtension();
        if (m_config.getVerbose())
        {
            std::cout << "-- Saving sample '" << i << " to " << filename << "." << std::endl;
        }
        matrix_io::eigen::write_matrix(filename, *p);
        ++i;
    }
}

void PredictSession::init()
{
    THROWERROR_ASSERT(m_has_config);
    THROWERROR_ASSERT(m_config.getTest());
    m_result = std::make_shared<Result>(m_config.getTest(), m_config.getNSamples());

    m_pos = m_stepfiles.rbegin();
    m_iter = 0;
    m_is_init = true;

    THROWERROR_ASSERT_MSG(m_config.getSavePrefix() != getModelRoot()->getPrefix(),
                          "Cannot have same prefix for model and predictions - both have " + m_config.getSavePrefix());

    if (m_config.getSaveFreq())
    {
        // create root file
        m_pred_rootfile = std::make_shared<RootFile>(m_config.getSavePrefix(), m_config.getSaveExtension(), true);
        m_pred_rootfile->createCsvStatusFile();
        m_pred_rootfile->flushLast();
    }

    if (m_config.getVerbose())
        info(std::cout, "");
{'fraction_non_alphanumeric': 0.13746384533414524, 'fraction_numerical': 0.0038057543005023597, 'mean_word_length': 3.5976207137858642, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6196818', 'n_tokens_mistral': 2321, 'n_tokens_neox': 2315, 'n_words': 344}