<commit_before>//=======================================================================
// Copyright (c) 2014-2015 <NAME>
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#ifndef DLL_RBM_TRAINER_HPP
#define DLL_RBM_TRAINER_HPP

#include <memory>

#include "cpp_utils/algorithm.hpp"

#include "decay_type.hpp"
#include "batch.hpp"
#include "layer_traits.hpp"

namespace dll {

enum class init_watcher_t { INIT };
constexpr const init_watcher_t init_watcher = init_watcher_t::INIT;

template<typename RBM, typename RW, typename Enable = void>
struct watcher_type {
    using watcher_t = typename RBM::desc::template watcher_t<RBM>;
};

template<typename RBM, typename RW>
struct watcher_type<RBM, RW, std::enable_if_t<cpp::not_u<std::is_void<RW>::value>::value>> {
    using watcher_t = RW;
};

/*!
 * \brief A generic trainer for Restricted Boltzmann Machine
 *
 * This trainer use the specified trainer of the RBM to perform unsupervised
 * training.
 */
template<typename RBM, bool EnableWatcher, typename RW>
struct rbm_trainer {
    using rbm_t = RBM;

    template<typename R,bool Denoising>
    using trainer_t = typename rbm_t::desc::template trainer_t<R,Denoising>;

    using watcher_t = typename watcher_type<rbm_t, RW>::watcher_t;

    mutable watcher_t watcher;

    rbm_trainer() : watcher() {}

    template<typename... Arg>
    rbm_trainer(init_watcher_t /*init*/, Arg... args) : watcher(args...) {}

    template<typename Iterator, cpp_enable_if_cst(layer_traits<rbm_t>::init_weights())>
    static void init_weights(RBM& rbm, Iterator first, Iterator last){
        rbm.init_weights(first, last);
    }

    template<typename Iterator, cpp_disable_if_cst(layer_traits<rbm_t>::init_weights())>
    static void init_weights(RBM&, Iterator, Iterator){
        //NOP
    }

    template<bool Denoising, typename IIterator, typename EIterator, cpp_enable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static void shuffle(IIterator ifirst, IIterator ilast, EIterator efirst, EIterator elast){
        static std::random_device rd;
        static std::mt19937_64 g(rd());

        if(Denoising){
            cpp::parallel_shuffle(ifirst, ilast, efirst, elast, g);
        } else {
            std::shuffle(ifirst, ilast, g);
        }
    }

    template<bool Denoising, typename IIterator, typename EIterator, cpp_disable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static void shuffle(IIterator, IIterator, EIterator, EIterator){}

    template<bool Denoising, typename IIterator, typename EIterator, typename IVector, typename EVector, cpp_enable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static auto prepare_it(IIterator ifirst, IIterator ilast, EIterator efirst, EIterator elast, IVector& ivec, EVector& evec){
        std::copy(ifirst, ilast, std::back_inserter(ivec));

        auto input_first = ivec.begin();
        auto input_last = ivec.end();

        if(Denoising){
            std::copy(efirst, elast, std::back_inserter(evec));

            auto expected_first = evec.begin();
            auto expected_last = evec.end();
            return std::make_tuple(input_first, input_last, expected_first, expected_last);
        } else {
            return std::make_tuple(input_first, input_last, input_first, input_last);
        }
    }

    template<bool Denoising, typename IIterator, typename EIterator, typename IVector, typename EVector, cpp_disable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static auto prepare_it(IIterator ifirst, IIterator ilast, EIterator efirst, EIterator elast, IVector&, EVector&){
        return std::make_tuple(ifirst, ilast, efirst, elast);
    }

    template<typename rbm_t, typename Iterator>
    using fix_iterator_t = std::conditional_t<
        layer_traits<rbm_t>::has_shuffle(),
        typename std::vector<typename std::iterator_traits<Iterator>::value_type>::iterator,
        Iterator>;

    std::size_t batch_size = 0;
    std::size_t total_batches = 0;
    typename rbm_t::weight last_error = 0.0;

    //Note: input_first/input_last only relevant for its size, not
    //values since they can point to the input of the first level
    //and not the current level
    template<bool Denoising = true, typename Iterator>
    void init_training(RBM& rbm, Iterator input_first, Iterator input_last){
        rbm.momentum = rbm.initial_momentum;

        if(EnableWatcher){
            watcher.training_begin(rbm);
        }

        //Compute the number of batches
        batch_size = get_batch_size(rbm);

        auto size = distance(input_first, input_last);

        //TODO Better handling of incomplete batch size would solve this problem (this could be done by
        //cleaning the data before the last batch)
        if(size % batch_size != 0){
            std::cout << "WARNING: The number of samples should be divisible by the batch size" << std::endl;
            std::cout << "         This may cause discrepancies in the results." << std::endl;
        }

        //Only used for debugging purposes, no need to be precise
        total_batches = size / batch_size;

        last_error = 0.0;
    }

    template<typename Iterator>
    typename rbm_t::weight train(RBM& rbm, Iterator first, Iterator last, std::size_t max_epochs){
        return train<false>(rbm, first, last, first, last, max_epochs);
    }<commit_msg>Less overhead for slow iterators<commit_after>//=======================================================================
// Copyright (c) 2014-2015 <NAME>
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#ifndef DLL_RBM_TRAINER_HPP
#define DLL_RBM_TRAINER_HPP

#include <memory>

#include "cpp_utils/algorithm.hpp"

#include "decay_type.hpp"
#include "batch.hpp"
#include "layer_traits.hpp"

namespace dll {

enum class init_watcher_t { INIT };
constexpr const init_watcher_t init_watcher = init_watcher_t::INIT;

template<typename RBM, typename RW, typename Enable = void>
struct watcher_type {
    using watcher_t = typename RBM::desc::template watcher_t<RBM>;
};

template<typename RBM, typename RW>
struct watcher_type<RBM, RW, std::enable_if_t<cpp::not_u<std::is_void<RW>::value>::value>> {
    using watcher_t = RW;
};

/*!
 * \brief A generic trainer for Restricted Boltzmann Machine
 *
 * This trainer use the specified trainer of the RBM to perform unsupervised
 * training.
 */
template<typename RBM, bool EnableWatcher, typename RW>
struct rbm_trainer {
    using rbm_t = RBM;

    template<typename R,bool Denoising>
    using trainer_t = typename rbm_t::desc::template trainer_t<R,Denoising>;

    using watcher_t = typename watcher_type<rbm_t, RW>::watcher_t;

    mutable watcher_t watcher;

    rbm_trainer() : watcher() {}

    template<typename... Arg>
    rbm_trainer(init_watcher_t /*init*/, Arg... args) : watcher(args...) {}

    template<typename Iterator, cpp_enable_if_cst(layer_traits<rbm_t>::init_weights())>
    static void init_weights(RBM& rbm, Iterator first, Iterator last){
        rbm.init_weights(first, last);
    }

    template<typename Iterator, cpp_disable_if_cst(layer_traits<rbm_t>::init_weights())>
    static void init_weights(RBM&, Iterator, Iterator){
        //NOP
    }

    template<bool Denoising, typename IIterator, typename EIterator, cpp_enable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static void shuffle(IIterator ifirst, IIterator ilast, EIterator efirst, EIterator elast){
        static std::random_device rd;
        static std::mt19937_64 g(rd());

        if(Denoising){
            cpp::parallel_shuffle(ifirst, ilast, efirst, elast, g);
        } else {
            std::shuffle(ifirst, ilast, g);
        }
    }

    template<bool Denoising, typename IIterator, typename EIterator, cpp_disable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static void shuffle(IIterator, IIterator, EIterator, EIterator){}

    template<bool Denoising, typename IIterator, typename EIterator, typename IVector, typename EVector, cpp_enable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static auto prepare_it(IIterator ifirst, IIterator ilast, EIterator efirst, EIterator elast, IVector& ivec, EVector& evec){
        std::copy(ifirst, ilast, std::back_inserter(ivec));

        auto input_first = ivec.begin();
        auto input_last = ivec.end();

        if(Denoising){
            std::copy(efirst, elast, std::back_inserter(evec));

            auto expected_first = evec.begin();
            auto expected_last = evec.end();
            return std::make_tuple(input_first, input_last, expected_first, expected_last);
        } else {
            return std::make_tuple(input_first, input_last, input_first, input_last);
        }
    }

    template<bool Denoising, typename IIterator, typename EIterator, typename IVector, typename EVector, cpp_disable_if_cst(layer_traits<rbm_t>::has_shuffle())>
    static auto prepare_it(IIterator ifirst, IIterator ilast, EIterator efirst, EIterator elast, IVector&, EVector&){
        return std::make_tuple(ifirst, ilast, efirst, elast);
    }

    template<typename rbm_t, typename Iterator>
    using fix_iterator_t = std::conditional_t<
        layer_traits<rbm_t>::has_shuffle(),
        typename std::vector<typename std::iterator_traits<Iterator>::value_type>::iterator,
        Iterator>;

    std::size_t batch_size = 0;
    std::size_t total_batches = 0;
    typename rbm_t::weight last_error = 0.0;

    //Note: input_first/input_last only relevant for its size, not
    //values since they can point to the input of the first level
    //and not the current level
    template<bool Denoising = true, typename Iterator>
    void init_training(RBM& rbm, Iterator input_first, Iterator input_last){
        rbm.momentum = rbm.initial_momentum;

        if(EnableWatcher){
            watcher.training_begin(rbm);
        }

        //Get the size of each batches
        batch_size = get_batch_size(rbm);

        if(std::is_same<typename std::iterator_traits<Iterator>::iterator_category, std::random_access_iterator_tag>::value){
            auto size = distance(input_first, input_last);

            //TODO Better handling of incomplete batch size would solve this problem (this could be done by
            //cleaning the data before the last batch)
            if(size % batch_size != 0){
                std::cout << "WARNING: The number of samples should be divisible by the batch size" << std::endl;
                std::cout << "         This may cause discrepancies in the results." << std::endl;
            }

            //Only used for debugging purposes, no need to be precise
            total_batches = size / batch_size;
        } else {
            total_batches = 0;
        }

        last_error = 0.0;
    }

    template<typename Iterator>
    typename rbm_t::weight train(RBM& rbm, Iterator first, Iterator last, std::size_t max_epochs){
        return train<false>(rbm, first, last, first, last, max_epochs);
    }
{'fraction_non_alphanumeric': 0.1283723423262462, 'fraction_numerical': 0.004020010720028586, 'mean_word_length': 3.6880234505862646, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '939702', 'n_tokens_mistral': 3651, 'n_tokens_neox': 3430, 'n_words': 883}