<commit_before>//=======================================================================
// Copyright <NAME> 2015.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <string>

#include "etl/etl.hpp"

#include "cpp_utils/parallel.hpp"

#include "dll/conv_rbm.hpp"
#include "dll/conv_rbm_mp.hpp"
#include "dll/dbn.hpp"
#include "dll/avgp_layer.hpp"
#include "dll/mp_layer.hpp"
#include "dll/ocv_visualizer.hpp"

#include "nice_svm.hpp"

#include "config.hpp"
#include "standard.hpp"
#include "utils.hpp"
#include "reports.hpp"
#include "dtw.hpp"        //Dynamic time warping

#define LOCAL_MEAN_SCALING
#include "scaling.hpp"      //Scaling functions

//The different configurations
#include "config_third.hpp"
#include "config_half.hpp"
#include "config_full.hpp"

#if defined(THIRD_CRBM_PMP_1) || defined(THIRD_CRBM_MP_1) || defined(THIRD_RBM_1)
#define THIRD_LEVELS 1
#endif

#if defined(THIRD_CRBM_PMP_2) || defined(THIRD_CRBM_MP_2) || defined(THIRD_RBM_2)
#define THIRD_LEVELS 2
#endif

#if defined(THIRD_CRBM_PMP_3) || defined(THIRD_CRBM_MP_3) || defined(THIRD_RBM_3)
#define THIRD_LEVELS 3
#endif

#if defined(HALF_CRBM_PMP_1) || defined(HALF_CRBM_MP_1)
#define HALF_LEVELS 1
#endif

#if defined(HALF_CRBM_PMP_2) || defined(HALF_CRBM_MP_2)
#define HALF_LEVELS 2
#endif

#if defined(HALF_CRBM_PMP_3) || defined(HALF_CRBM_MP_3)
#define HALF_LEVELS 3
#endif

#if defined(FULL_CRBM_PMP_1) || defined(FULL_CRBM_MP_1)
#define FULL_LEVELS 1
#endif

#if defined(FULL_CRBM_PMP_2) || defined(FULL_CRBM_MP_2)
#define FULL_LEVELS 2
#endif

#if defined(FULL_CRBM_PMP_3) || defined(FULL_CRBM_MP_3)
#define FULL_LEVELS 3
#endif

#if !defined(HALF_LEVELS) || !defined(THIRD_LEVELS) || !defined(FULL_LEVELS)
static_assert(false, "Invalid configuration");
#endif

namespace {

struct patch_iterator : std::iterator<std::input_iterator_tag, etl::dyn_matrix<weight, 3>> {
    config& conf;
    const washington_dataset& dataset;
    const std::vector<std::string>& image_names;

    std::size_t current_image = 0;
    std::vector<etl::dyn_matrix<weight, 3>> patches;
    std::size_t current_patch = 0;

    patch_iterator(config& conf, const washington_dataset& dataset, const std::vector<std::string>& image_names, std::size_t i = 0)
            : conf(conf), dataset(dataset), image_names(image_names), current_image(i) {
        if(current_image < image_names.size()){
            patches = mat_to_patches(conf, dataset.word_images.at(image_names[current_image]), true);
        }
    }

    patch_iterator(const patch_iterator& rhs) = default;
    patch_iterator& operator=(const patch_iterator& rhs) = default;

    bool operator==(const patch_iterator& rhs){
        if(current_image == image_names.size() && current_image == rhs.current_image){
            return true;
        } else {
            return current_image == rhs.current_image && current_patch == rhs.current_patch;
        }
    }

    bool operator!=(const patch_iterator& rhs){
        return !(*this == rhs);
    }
    etl::dyn_matrix<weight, 3>& operator*(){
        return patches[current_patch];
    }

    etl::dyn_matrix<weight, 3>* operator->(){
        return &patches[current_patch];
    }

    patch_iterator operator++(){
        if(current_patch == patches.size() - 1){
            ++current_image;
            current_patch = 0;

            if(current_image < image_names.size()){
                patches = mat_to_patches(conf, dataset.word_images.at(image_names[current_image]), true);
            }
        } else {
            ++current_patch;
        }

        return *this;
    }

    patch_iterator operator++(int){
        patch_iterator it = *this;
        ++(*this);
        return it;
    }
};

template<typename Dataset, typename Set, typename DBN>
void evaluate_patches(const Dataset& dataset, const Set& set, config& conf, const DBN& dbn, const std::vector<std::string>& train_word_names, const std::vector<std::string>& test_image_names, bool training){
    //Get some sizes

    const std::size_t patch_height = HEIGHT / conf.downscale;
    const std::size_t patch_width = conf.patch_width;

    auto result_folder = select_folder("./results/");

    generate_rel_files(result_folder, dataset, set, test_image_names);

    std::cout << "Prepare the outputs ..." << std::endl;

    std::vector<std::vector<typename DBN::output_t>> test_features_a(test_image_names.size());

    cpp::default_thread_pool<> pool;

    cpp::parallel_foreach_i(pool, test_image_names.begin(), test_image_names.end(),
        [&,patch_height,patch_width](auto& test_image, std::size_t i){
            auto& vec = test_features_a[i];

            auto patches = mat_to_patches(conf, dataset.word_images.at(test_image), false);

            for(auto& patch : patches){
                vec.push_back(dbn.prepare_one_output());
                dbn.activation_probabilities(patch, vec.back());
            }

#ifdef LOCAL_LINEAR_SCALING
            local_linear_feature_scaling(vec);
#endif

#ifdef LOCAL_MEAN_SCALING
            local_mean_feature_scaling(vec);
#endif
        });

#ifdef GLOBAL_MEAN_SCALING
    auto scale = global_mean_scaling(test_features_a, conf, training);
#endif

#ifdef GLOBAL_LINEAR_SCALING
    auto scale = global_linear_scaling(test_features_a, conf, training);
#endif

#if defined(GLOBAL_MEAN_SCALING) || defined(GLOBAL_LINEAR_SCALING)
    for(std::size_t t = 0; t < test_features_a.size(); ++t){
        for(std::size_t i = 0; i < test_features_a[t].size(); ++i){
            for(std::size_t f = 0; f < test_features_a.back().back().size(); ++f){
                test_features_a[t][i][f] = scale(test_features_a[t][i][f], conf.scale_a[f], conf.scale_b[f]);
            }
        }
    }
#else
    cpp_unused(training);
    cpp_unused(conf);
#endif

    std::cout << "... done" << std::endl;

    std::cout << "Evaluate performance..." << std::endl;

    std::size_t evaluated = 0;

    std::vector<double> eer(set.keywords.size());
    std::vector<double> ap(set.keywords.size());

    std::ofstream global_top_stream(result_folder + "/global_top_file");
    std::ofstream local_top_stream(result_folder + "/local_top_file");<commit_msg>Refactorings<commit_after>//=======================================================================
// Copyright <NAME> 2015.
// Distributed under the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <string>

#include "etl/etl.hpp"

#include "cpp_utils/parallel.hpp"

#include "dll/conv_rbm.hpp"
#include "dll/conv_rbm_mp.hpp"
#include "dll/dbn.hpp"
#include "dll/avgp_layer.hpp"
#include "dll/mp_layer.hpp"
#include "dll/ocv_visualizer.hpp"

#include "nice_svm.hpp"

#include "config.hpp"
#include "standard.hpp"
#include "utils.hpp"
#include "reports.hpp"
#include "dtw.hpp"        //Dynamic time warping

#define LOCAL_MEAN_SCALING
#include "scaling.hpp"      //Scaling functions

//The different configurations
#include "config_third.hpp"
#include "config_half.hpp"
#include "config_full.hpp"

#if defined(THIRD_CRBM_PMP_1) || defined(THIRD_CRBM_MP_1) || defined(THIRD_RBM_1)
#define THIRD_LEVELS 1
#endif

#if defined(THIRD_CRBM_PMP_2) || defined(THIRD_CRBM_MP_2) || defined(THIRD_RBM_2)
#define THIRD_LEVELS 2
#endif

#if defined(THIRD_CRBM_PMP_3) || defined(THIRD_CRBM_MP_3) || defined(THIRD_RBM_3)
#define THIRD_LEVELS 3
#endif

#if defined(HALF_CRBM_PMP_1) || defined(HALF_CRBM_MP_1)
#define HALF_LEVELS 1
#endif

#if defined(HALF_CRBM_PMP_2) || defined(HALF_CRBM_MP_2)
#define HALF_LEVELS 2
#endif

#if defined(HALF_CRBM_PMP_3) || defined(HALF_CRBM_MP_3)
#define HALF_LEVELS 3
#endif

#if defined(FULL_CRBM_PMP_1) || defined(FULL_CRBM_MP_1)
#define FULL_LEVELS 1
#endif

#if defined(FULL_CRBM_PMP_2) || defined(FULL_CRBM_MP_2)
#define FULL_LEVELS 2
#endif

#if defined(FULL_CRBM_PMP_3) || defined(FULL_CRBM_MP_3)
#define FULL_LEVELS 3
#endif

#if !defined(HALF_LEVELS) || !defined(THIRD_LEVELS) || !defined(FULL_LEVELS)
static_assert(false, "Invalid configuration");
#endif

namespace {

struct patch_iterator : std::iterator<std::input_iterator_tag, etl::dyn_matrix<weight, 3>> {
    config& conf;
    const washington_dataset& dataset;
    const std::vector<std::string>& image_names;

    std::size_t current_image = 0;
    std::vector<etl::dyn_matrix<weight, 3>> patches;
    std::size_t current_patch = 0;

    patch_iterator(config& conf, const washington_dataset& dataset, const std::vector<std::string>& image_names, std::size_t i = 0)
            : conf(conf), dataset(dataset), image_names(image_names), current_image(i) {
        if(current_image < image_names.size()){
            patches = mat_to_patches(conf, dataset.word_images.at(image_names[current_image]), true);
        }
    }

    patch_iterator(const patch_iterator& rhs) = default;
    patch_iterator& operator=(const patch_iterator& rhs) = default;

    bool operator==(const patch_iterator& rhs){
        if(current_image == image_names.size() && current_image == rhs.current_image){
            return true;
        } else {
            return current_image == rhs.current_image && current_patch == rhs.current_patch;
        }
    }

    bool operator!=(const patch_iterator& rhs){
        return !(*this == rhs);
    }
    etl::dyn_matrix<weight, 3>& operator*(){
        return patches[current_patch];
    }

    etl::dyn_matrix<weight, 3>* operator->(){
        return &patches[current_patch];
    }

    patch_iterator operator++(){
        if(current_patch == patches.size() - 1){
            ++current_image;
            current_patch = 0;

            if(current_image < image_names.size()){
                patches = mat_to_patches(conf, dataset.word_images.at(image_names[current_image]), true);
            }
        } else {
            ++current_patch;
        }

        return *this;
    }

    patch_iterator operator++(int){
        patch_iterator it = *this;
        ++(*this);
        return it;
    }
};

struct parameters {
    double sc_band;
};

template<typename Dataset, typename DBN>
std::vector<std::vector<typename DBN::output_t>> prepare_outputs(
        cpp::default_thread_pool<>& pool, const Dataset& dataset, const DBN& dbn, config& conf,
        const std::vector<std::string>& test_image_names, bool training){
    //Get some sizes
    const std::size_t patch_height = HEIGHT / conf.downscale;
    const std::size_t patch_width = conf.patch_width;

    std::vector<std::vector<typename DBN::output_t>> test_features_a(test_image_names.size());

    std::cout << "Prepare the outputs ..." << std::endl;

    cpp::parallel_foreach_i(pool, test_image_names.begin(), test_image_names.end(),
        [&,patch_height,patch_width](auto& test_image, std::size_t i){
            auto& vec = test_features_a[i];

            auto patches = mat_to_patches(conf, dataset.word_images.at(test_image), false);

            for(auto& patch : patches){
                vec.push_back(dbn.prepare_one_output());
                dbn.activation_probabilities(patch, vec.back());
            }

#ifdef LOCAL_LINEAR_SCALING
            local_linear_feature_scaling(vec);
#endif

#ifdef LOCAL_MEAN_SCALING
            local_mean_feature_scaling(vec);
#endif
        });

#ifdef GLOBAL_MEAN_SCALING
    auto scale = global_mean_scaling(test_features_a, conf, training);
#endif

#ifdef GLOBAL_LINEAR_SCALING
    auto scale = global_linear_scaling(test_features_a, conf, training);
#endif

#if defined(GLOBAL_MEAN_SCALING) || defined(GLOBAL_LINEAR_SCALING)
    for(std::size_t t = 0; t < test_features_a.size(); ++t){
        for(std::size_t i = 0; i < test_features_a[t].size(); ++i){
            for(std::size_t f = 0; f < test_features_a.back().back().size(); ++f){
                test_features_a[t][i][f] = scale(test_features_a[t][i][f], conf.scale_a[f], conf.scale_b[f]);
            }
        }
    }
#else
    cpp_unused(training);
    cpp_unused(conf);
#endif

    std::cout << "... done" << std::endl;

    return test_features_a;
}

template<typename Dataset, typename Set, typename DBN>
void evaluate_patches(const Dataset& dataset, const Set& set, config& conf, const DBN& dbn, const std::vector<std::string>& train_word_names, const std::vector<std::string>& test_image_names, bool training){

    auto result_folder = select_folder("./results/");

    generate_rel_files(result_folder, dataset, set, test_image_names);

    cpp::default_thread_pool<> pool;

    auto test_features_a = prepare_outputs(pool, dataset, dbn, conf, test_image_names, training);

    std::cout << "Evaluate performance..." << std::endl;

    std::size_t evaluated = 0;

    std::vector<double> eer(set.keywords.size());
    std::vector<double> ap(set.keywords.size());

    std::ofstream global_top_stream(result_folder + "/global_top_file");
    std::ofstream local_top_stream(result_folder + "/local_top_file");
{'fraction_non_alphanumeric': 0.15303935078854694, 'fraction_numerical': 0.0071964477109171645, 'mean_word_length': 4.055340557275541, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6091270', 'n_tokens_mistral': 4580, 'n_tokens_neox': 4390, 'n_words': 839}