<commit_before>//=======================================================================
// Copyright (c) 2014-2015 <NAME>
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

/*!
 * \file Stochastic Gradient Descent (SGD) Implementation for Dense networks
 */

#ifndef DLL_DENSE_STOCHASTIC_GRADIENT_DESCENT
#define DLL_DENSE_STOCHASTIC_GRADIENT_DESCENT

#include "cpp_utils/static_if.hpp"
#include "context.hpp"

namespace dll {

template<typename DBN, typename Layer, typename Enable = void>
struct dense_sgd_context;

template<typename DBN, typename Layer>
struct dense_sgd_context <DBN, Layer, std::enable_if_t<layer_traits<Layer>::is_dense_layer()>> {
    using layer_t = Layer;
    using dbn_t = DBN;
    using weight = typename layer_t::weight;

    static constexpr const auto num_visible = layer_t::num_visible;
    static constexpr const auto num_hidden = layer_t::num_hidden;

    static constexpr const auto batch_size = dbn_t::batch_size;

    etl::fast_matrix<weight, num_visible, num_hidden> w_grad;
    etl::fast_matrix<weight, num_hidden> b_grad;

    etl::fast_matrix<weight, num_visible, num_hidden> w_inc;
    etl::fast_matrix<weight, num_hidden> b_inc;

    etl::fast_matrix<weight, batch_size, num_hidden> output;
    etl::fast_matrix<weight, batch_size, num_hidden> errors;

    dense_sgd_context() : w_inc(0.0), b_inc(0.0), output(0.0), errors(0.0) {}
};

template<typename DBN, typename Layer>
struct dense_sgd_context <DBN, Layer, std::enable_if_t<layer_traits<Layer>::is_convolutional_layer()>> {
    using layer_t = Layer;
    using dbn_t = DBN;
    using weight = typename layer_t::weight;

    static constexpr const std::size_t NV1 = layer_t::NV1;
    static constexpr const std::size_t NV2 = layer_t::NV2;
    static constexpr const std::size_t NH1 = layer_t::NH1;
    static constexpr const std::size_t NH2 = layer_t::NH2;
    static constexpr const std::size_t NW1 = layer_t::NW1;
    static constexpr const std::size_t NW2 = layer_t::NW2;
    static constexpr const std::size_t NC = layer_t::NC;
    static constexpr const std::size_t K = layer_t::K;

    static constexpr const auto batch_size = dbn_t::batch_size;

    etl::fast_matrix<weight, NC, K, NW1, NW2> w_grad;
    etl::fast_matrix<weight, K> b_grad;

    etl::fast_matrix<weight, NC, K, NW1, NW2> w_inc;
    etl::fast_matrix<weight, K> b_inc;

    etl::fast_matrix<weight, batch_size, K, NH1, NH2> output;
    etl::fast_matrix<weight, batch_size, K, NH1, NH2> errors;

    dense_sgd_context() : w_inc(0.0), b_inc(0.0), output(0.0), errors(0.0) {}
};

template<typename DBN>
struct dense_sgd_trainer {
    using dbn_t = DBN;
    using weight = typename dbn_t::weight;
    using this_type = dense_sgd_trainer<dbn_t>;

    using context_tuple_t = typename dbn_context_builder<dense_sgd_context, dbn_t, typename dbn_t::tuple_type>::type;

    static constexpr const auto layers = dbn_t::layers;
    static constexpr const auto batch_size = dbn_t::batch_size;

    dbn_t& dbn;
    typename dbn_t::tuple_type& tuples;

    context_tuple_t contexts;

    dense_sgd_trainer(dbn_t& dbn) : dbn(dbn), tuples(dbn.tuples) {
        //Nothing else to init
    }

    void init_training(std::size_t){}

    template<typename Sample>
    void compute_outputs(const Sample& item_data){
        auto& first_layer = dbn.template layer_get<0>();
        auto& first_layer_context = std::get<0>(contexts);

        first_layer.batch_activate_hidden(first_layer_context.output, item_data);

        cpp::for_each_pair(tuples, contexts, [](auto&, auto& layer_2, auto& ctx1, auto& ctx2){
            layer_2.batch_activate_hidden(ctx2.output, ctx1.output);
        });
    }

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() == 2)>
    static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        dense_compute_weight_gradients<Layer, Weight>(grad, inputs, errors);
    }

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() != 2)>
    static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        //TODO By improving reshape to be variadic, this could be
        //made a lot faster

        constexpr const auto Batch = etl::decay_traits<Inputs>::template dim<0>();
        typename Layer::template input_batch_t<Batch> input;

        for(std::size_t b = 0; b < Batch; ++b){
            input(b) = inputs(b);
        }

        dense_compute_weight_gradients<Layer, Weight>(grad, input, errors);
    }

#ifndef ETL_BLAS_MODE

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() == 2)>
    static void dense_compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        for(std::size_t i = 0; i < batch_size; ++i){
            grad += etl::outer(inputs(i), errors(i));
        }
    }

#else

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && std::is_same<Weight, float>::value)>
    static void dense_compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        cpp_unused(grad);
        cpp_unused(inputs);
        cpp_unused(errors);
        for(std::size_t i = 0; i < batch_size; ++i){
            cblas_sger(
                CblasRowMajor,
                etl::dim<1>(inputs), etl::dim<1>(errors),
                1.0f,
                inputs(i).memory_start(), 1,
                errors(i).memory_start(), 1,
                grad.memory_start(), etl::dim<1>(errors)
            );
        }
    }

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && std::is_same<Weight, double>::value)>
    static void dense_compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        cpp_unused(grad);
        cpp_unused(inputs);
        cpp_unused(errors);
        for(std::size_t i = 0; i < batch_size; ++i){
            cblas_dger(
                CblasRowMajor,
                etl::dim<1>(inputs), etl::dim<1>(errors),
                1.0,
                inputs(i).memory_start(), 1,
                errors(i).memory_start(), 1,
                grad.memory_start(), etl::dim<1>(errors)
            );
        }
    }

#endif

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_convolutional_layer())>
    static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        constexpr const auto K = Layer::K;
        constexpr const auto NC = Layer::NC;
        constexpr const auto NW1 = Layer::NW1;
        constexpr const auto NW2 = Layer::NW2;
        constexpr const auto NH1 = Layer::NH1;
        constexpr const auto NH2 = Layer::NH2;

        cpp_unused(grad);
        cpp_unused(inputs);
        cpp_unused(errors);

        for(std::size_t b = 0; b < batch_size; ++b){
            for(std::size_t c = 0; c < NC; ++c){
                for(std::size_t k = 0; k < K; ++k){
                    for(std::size_t i = 0; i < NW1; ++i){
                        for(std::size_t j = 0; j < NW2; ++j){
                            for(std::size_t ii = 0; ii < NH1; ++ii){
                                for(std::size_t jj = 0; jj < NH2; ++jj){
                                    grad(c, k, i, j) += errors(b, k, ii, jj) * inputs(b, c, i + ii, j + jj);
                                }
                            }
                        }
                    }
                }
            }
        }
    }<commit_msg>Clean<commit_after>//=======================================================================
// Copyright (c) 2014-2015 <NAME>
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

/*!
 * \file Stochastic Gradient Descent (SGD) Implementation for Dense networks
 */

#ifndef DLL_DENSE_STOCHASTIC_GRADIENT_DESCENT
#define DLL_DENSE_STOCHASTIC_GRADIENT_DESCENT

#include "cpp_utils/static_if.hpp"
#include "context.hpp"

namespace dll {

template<typename DBN, typename Layer, typename Enable = void>
struct dense_sgd_context;

template<typename DBN, typename Layer>
struct dense_sgd_context <DBN, Layer, std::enable_if_t<layer_traits<Layer>::is_dense_layer()>> {
    using layer_t = Layer;
    using dbn_t = DBN;
    using weight = typename layer_t::weight;

    static constexpr const auto num_visible = layer_t::num_visible;
    static constexpr const auto num_hidden = layer_t::num_hidden;

    static constexpr const auto batch_size = dbn_t::batch_size;

    etl::fast_matrix<weight, num_visible, num_hidden> w_grad;
    etl::fast_matrix<weight, num_hidden> b_grad;

    etl::fast_matrix<weight, num_visible, num_hidden> w_inc;
    etl::fast_matrix<weight, num_hidden> b_inc;

    etl::fast_matrix<weight, batch_size, num_hidden> output;
    etl::fast_matrix<weight, batch_size, num_hidden> errors;

    dense_sgd_context() : w_inc(0.0), b_inc(0.0), output(0.0), errors(0.0) {}
};

template<typename DBN, typename Layer>
struct dense_sgd_context <DBN, Layer, std::enable_if_t<layer_traits<Layer>::is_convolutional_layer()>> {
    using layer_t = Layer;
    using dbn_t = DBN;
    using weight = typename layer_t::weight;

    static constexpr const std::size_t NV1 = layer_t::NV1;
    static constexpr const std::size_t NV2 = layer_t::NV2;
    static constexpr const std::size_t NH1 = layer_t::NH1;
    static constexpr const std::size_t NH2 = layer_t::NH2;
    static constexpr const std::size_t NW1 = layer_t::NW1;
    static constexpr const std::size_t NW2 = layer_t::NW2;
    static constexpr const std::size_t NC = layer_t::NC;
    static constexpr const std::size_t K = layer_t::K;

    static constexpr const auto batch_size = dbn_t::batch_size;

    etl::fast_matrix<weight, NC, K, NW1, NW2> w_grad;
    etl::fast_matrix<weight, K> b_grad;

    etl::fast_matrix<weight, NC, K, NW1, NW2> w_inc;
    etl::fast_matrix<weight, K> b_inc;

    etl::fast_matrix<weight, batch_size, K, NH1, NH2> output;
    etl::fast_matrix<weight, batch_size, K, NH1, NH2> errors;

    dense_sgd_context() : w_inc(0.0), b_inc(0.0), output(0.0), errors(0.0) {}
};

template<typename DBN>
struct dense_sgd_trainer {
    using dbn_t = DBN;
    using weight = typename dbn_t::weight;
    using this_type = dense_sgd_trainer<dbn_t>;

    using context_tuple_t = typename dbn_context_builder<dense_sgd_context, dbn_t, typename dbn_t::tuple_type>::type;

    static constexpr const auto layers = dbn_t::layers;
    static constexpr const auto batch_size = dbn_t::batch_size;

    dbn_t& dbn;
    typename dbn_t::tuple_type& tuples;

    context_tuple_t contexts;

    dense_sgd_trainer(dbn_t& dbn) : dbn(dbn), tuples(dbn.tuples) {
        //Nothing else to init
    }

    void init_training(std::size_t){}

    template<typename Sample>
    void compute_outputs(const Sample& item_data){
        auto& first_layer = dbn.template layer_get<0>();
        auto& first_layer_context = std::get<0>(contexts);

        first_layer.batch_activate_hidden(first_layer_context.output, item_data);

        cpp::for_each_pair(tuples, contexts, [](auto&, auto& layer_2, auto& ctx1, auto& ctx2){
            layer_2.batch_activate_hidden(ctx2.output, ctx1.output);
        });
    }

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() == 2)>
    static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        dense_compute_weight_gradients<Layer, Weight>(grad, inputs, errors);
    }

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() != 2)>
    static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        //TODO By improving reshape to be variadic, this could be
        //made a lot faster

        constexpr const auto Batch = etl::decay_traits<Inputs>::template dim<0>();
        typename Layer::template input_batch_t<Batch> input;

        for(std::size_t b = 0; b < Batch; ++b){
            input(b) = inputs(b);
        }

        dense_compute_weight_gradients<Layer, Weight>(grad, input, errors);
    }

#ifndef ETL_BLAS_MODE

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && etl::decay_traits<Inputs>::dimensions() == 2)>
    static void dense_compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        for(std::size_t i = 0; i < batch_size; ++i){
            grad += etl::outer(inputs(i), errors(i));
        }
    }

#else

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && std::is_same<Weight, float>::value)>
    static void dense_compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        for(std::size_t i = 0; i < batch_size; ++i){
            cblas_sger(
                CblasRowMajor,
                etl::dim<1>(inputs), etl::dim<1>(errors),
                1.0f,
                inputs(i).memory_start(), 1,
                errors(i).memory_start(), 1,
                grad.memory_start(), etl::dim<1>(errors)
            );
        }
    }

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_dense_layer() && std::is_same<Weight, double>::value)>
    static void dense_compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        for(std::size_t i = 0; i < batch_size; ++i){
            cblas_dger(
                CblasRowMajor,
                etl::dim<1>(inputs), etl::dim<1>(errors),
                1.0,
                inputs(i).memory_start(), 1,
                errors(i).memory_start(), 1,
                grad.memory_start(), etl::dim<1>(errors)
            );
        }
    }

#endif

    template<typename Layer, typename Weight, typename Grad, typename Inputs, typename Errors, cpp_enable_if(decay_layer_traits<Layer>::is_convolutional_layer())>
    static void compute_weight_gradients(Grad& grad, Inputs& inputs, Errors& errors){
        constexpr const auto K = Layer::K;
        constexpr const auto NC = Layer::NC;
        constexpr const auto NW1 = Layer::NW1;
        constexpr const auto NW2 = Layer::NW2;
        constexpr const auto NH1 = Layer::NH1;
        constexpr const auto NH2 = Layer::NH2;

        for(std::size_t b = 0; b < batch_size; ++b){
            for(std::size_t c = 0; c < NC; ++c){
                for(std::size_t k = 0; k < K; ++k){
                    for(std::size_t i = 0; i < NW1; ++i){
                        for(std::size_t j = 0; j < NW2; ++j){
                            for(std::size_t ii = 0; ii < NH1; ++ii){
                                for(std::size_t jj = 0; jj < NH2; ++jj){
                                    grad(c, k, i, j) += errors(b, k, ii, jj) * inputs(b, c, i + ii, j + jj);
                                }
                            }
                        }
                    }
                }
            }
        }
    }
{'fraction_non_alphanumeric': 0.13729418393539097, 'fraction_numerical': 0.01164465034746134, 'mean_word_length': 3.1915507740750457, 'pattern_counts': {'":': 0, '<': 137, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '669238', 'n_tokens_mistral': 5545, 'n_tokens_neox': 5132, 'n_words': 1197}