<commit_before>#include "dense_feature.h"
#include "evaluator.h"
#include "fast_sparse_feature.h"
#include "gheap.h"
#include "log_trivial.h"
#include "metric.h"
#include "ranking_cost_function.h"
#include "regression_cost_function.h"
#include "sparse_feature.h"
#include "util.h"
#include "workflow.h"

#include <boost/log/core.hpp>
#include <boost/log/expressions.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include <boost/log/sinks/sync_frontend.hpp>
#include <boost/log/sinks/text_ostream_backend.hpp>
#include <boost/log/sources/logger.hpp>
#include <boost/log/sources/record_ostream.hpp>
#include <random>
#include <thread>

class InMemoryBucketsProvider : public BucketsProvider
{
private:
    Workflow * workflow;
public:
    InMemoryBucketsProvider(Workflow * workflow) {
        this->workflow = workflow;
    }
    virtual BucketsCollection * get_buckets(uint32_t feature_id) {
        return workflow->get_trainer()->get_feature(feature_id)->get_buckets();
    }
};


class ColumnConsumerProviderForTraining : public ColumnConsumerProvider
{
public:
    RawFeatureType default_type;
    std::unique_ptr<RawFeature<float_t>> label_consumer;
    std::unique_ptr<QueryColumnConsumer> query_consumer;
    std::vector<std::unique_ptr<DynamicRawFeature>> features;

    virtual ColumnConsumer* create_label_consumer()
    {
        assert(label_consumer.get() == nullptr);
        label_consumer = std::unique_ptr<RawFeature<float_t>>(new RawFeature<float_t>());
        return label_consumer.get();
    }
    virtual ColumnConsumer*create_query_consumer()
    {
        assert(query_consumer.get() == nullptr);
        query_consumer = std::unique_ptr<QueryColumnConsumer>(new QueryColumnConsumer());
        return query_consumer.get();
    }
    virtual ColumnConsumer* create_feature_consumer()
    {
        std::unique_ptr<DynamicRawFeature> drf(new DynamicRawFeature(0, this->default_type));
        DynamicRawFeature * result = drf.get();
        features.push_back(std::move(drf));
        return result;
    }
};


Workflow::Workflow(Options & options) : options(options)
{
}

void Workflow::run()
{
    this->init();
    if (this->options.train) {
        this->run_train();
        return;
    }
    if (this->options.evaluate) {
        this->run_evaluate();
        return;
    }
    throw std::runtime_error("Unknown action.");
}

void Workflow::run_train()
{
this->tree_writer = this->get_tree_writer();
    this->cost_function = this->get_cost_function();
    this->tree_writer->set_cost_function(cost_function->get_registry_name());
    auto training_data = this->read_tsv();
    const std::vector<float_t> * labels = &std::get<0>(training_data);
    const std::vector<DOC_ID> * query_limits = &std::get<1>(training_data);
    FEATURE_PIPELINE_PTR_TYPE  features = std::move(std::get<2>(training_data));
    this->trainer = this->create_trainer(labels, query_limits, std::move(features));
    this->trainer->set_cost_function(std::move(cost_function));
    this->train_ensemble();
    this->tree_writer->close();
}

void Workflow::run_evaluate()
{
    this->ensemble = load_ensemble(this->options.input_tree);
    std::string metric_name = this->options.metric;
    if (metric_name.size() == 0) {
        metric_name = CostFunction::create(ensemble->get_cost_function())->get_default_metric_name();
    }
    std::unique_ptr<Metric> metric = std::unique_ptr<Metric>(Metric::get_metric(metric_name));

    INPUT_ROW_PIPELINE_PTR_TYPE input_pipe= std::shared_ptr<INPUT_ROW_PIPELINE_TYPE>(new INPUT_ROW_PIPELINE_TYPE(this->get_bbq_size()));
    std::shared_ptr<ColumnConsumerProvider> ccp(new ColumnConsumerProviderForEvaluation(input_pipe, ensemble.get(), this->options.exponentiate_label));
    std::unique_ptr<DataFileReader> tsv = this->get_tsv_reader(ccp, metric->is_query_based());

    std::thread reader(
         [tsv = std::move(tsv)]() mutable {
        tsv->read();
    });
    reader.detach();

    std::unique_ptr<std::ostream> epochs;<commit_msg>Minor fix<commit_after>#include "dense_feature.h"
#include "evaluator.h"
#include "fast_sparse_feature.h"
#include "gheap.h"
#include "log_trivial.h"
#include "metric.h"
#include "ranking_cost_function.h"
#include "regression_cost_function.h"
#include "sparse_feature.h"
#include "util.h"
#include "workflow.h"

#include <boost/log/core.hpp>
#include <boost/log/expressions.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include <boost/log/sinks/sync_frontend.hpp>
#include <boost/log/sinks/text_ostream_backend.hpp>
#include <boost/log/sources/logger.hpp>
#include <boost/log/sources/record_ostream.hpp>
#include <random>
#include <thread>

class InMemoryBucketsProvider : public BucketsProvider
{
private:
    Workflow * workflow;
public:
    InMemoryBucketsProvider(Workflow * workflow) {
        this->workflow = workflow;
    }
    virtual BucketsCollection * get_buckets(uint32_t feature_id) {
        return workflow->get_trainer()->get_feature(feature_id)->get_buckets();
    }
};


class ColumnConsumerProviderForTraining : public ColumnConsumerProvider
{
public:
    RawFeatureType default_type;
    std::unique_ptr<RawFeature<float_t>> label_consumer;
    std::unique_ptr<QueryColumnConsumer> query_consumer;
    std::vector<std::unique_ptr<DynamicRawFeature>> features;

    virtual ColumnConsumer* create_label_consumer()
    {
        assert(label_consumer.get() == nullptr);
        label_consumer = std::unique_ptr<RawFeature<float_t>>(new RawFeature<float_t>());
        return label_consumer.get();
    }
    virtual ColumnConsumer*create_query_consumer()
    {
        assert(query_consumer.get() == nullptr);
        query_consumer = std::unique_ptr<QueryColumnConsumer>(new QueryColumnConsumer());
        return query_consumer.get();
    }
    virtual ColumnConsumer* create_feature_consumer()
    {
        std::unique_ptr<DynamicRawFeature> drf(new DynamicRawFeature(0, this->default_type));
        DynamicRawFeature * result = drf.get();
        features.push_back(std::move(drf));
        return result;
    }
};


Workflow::Workflow(Options & options) : options(options)
{
}

void Workflow::run()
{
    this->init();
    if (this->options.train) {
        this->run_train();
        return;
    }
    if (this->options.evaluate) {
        this->run_evaluate();
        return;
    }
    throw std::runtime_error("Unknown action.");
}

void Workflow::run_train()
{
this->tree_writer = this->get_tree_writer();
    this->cost_function = this->get_cost_function();
    this->tree_writer->set_cost_function(cost_function->get_registry_name());
    auto training_data = this->read_tsv();
    const std::vector<float_t> * labels = &std::get<0>(training_data);
    const std::vector<DOC_ID> * query_limits = &std::get<1>(training_data);
    FEATURE_PIPELINE_PTR_TYPE  features = std::move(std::get<2>(training_data));
    this->trainer = this->create_trainer(labels, query_limits, std::move(features));
    this->trainer->set_cost_function(std::move(cost_function));
    this->train_ensemble();
    this->tree_writer->close();
}

void Workflow::run_evaluate()
{
    this->ensemble = load_ensemble(this->options.input_tree);
    std::string metric_name = this->options.metric;
    if (metric_name.size() == 0) {
        metric_name = CostFunction::create(ensemble->get_cost_function())->get_default_metric_name();
    }
    std::unique_ptr<Metric> metric = std::unique_ptr<Metric>(Metric::get_metric(metric_name));

    INPUT_ROW_PIPELINE_PTR_TYPE input_pipe= std::shared_ptr<INPUT_ROW_PIPELINE_TYPE>(new INPUT_ROW_PIPELINE_TYPE(this->get_bbq_size()));
    std::shared_ptr<ColumnConsumerProvider> ccp(new ColumnConsumerProviderForEvaluation(input_pipe, ensemble.get(), this->options.exponentiate_label));
    std::unique_ptr<DataFileReader> tsv = this->get_tsv_reader(ccp, metric->is_query_based());

    async_fill_pipeline(input_pipe,
         [tsv = std::move(tsv)]() mutable {
        tsv->read();
    });

    std::unique_ptr<std::ostream> epochs;
{'fraction_non_alphanumeric': 0.14028732042473455, 'fraction_numerical': 0.0017489069331667708, 'mean_word_length': 4.882439382806759, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 135, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4286426', 'n_tokens_mistral': 2653, 'n_tokens_neox': 2582, 'n_words': 372}