<commit_before>//=======================================================================
// Copyright (c) 2014-2020 <NAME>
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include <fstream>

#include "cpp_utils/assert.hpp" //Assertions
#include "cpp_utils/io.hpp"     // For binary writing

#include "etl/etl.hpp"

#include "layer.hpp"
#include "layer_traits.hpp"
#include "util/tmp.hpp"

namespace dll {

/*!
 * \brief Base class for RNN layers (fast / dynamic)
 */
template <typename Derived, typename Desc>
struct base_rnn_layer : layer<Derived> {
    using desc      = Desc;                            ///< The descriptor of the layer
    using derived_t = Derived;                         ///< The derived type (CRTP)
    using weight    = typename desc::weight;           ///< The data type for this layer
    using this_type = base_rnn_layer<derived_t, desc>; ///< The type of this layer
    using base_type = layer<Derived>;                  ///< The base type

    static constexpr auto activation_function = desc::activation_function; ///< The layer's activation function

    /*!
     * \brief Initialize the neural layer
     */
    base_rnn_layer()
            : base_type() {
        // Nothing to init here
    }

    base_rnn_layer(const base_rnn_layer& rhs) = delete;
    base_rnn_layer(base_rnn_layer&& rhs)      = delete;

    base_rnn_layer& operator=(const base_rnn_layer& rhs) = delete;
    base_rnn_layer& operator=(base_rnn_layer&& rhs) = delete;

    mutable etl::dyn_matrix<float, 3> x_t;
    mutable etl::dyn_matrix<float, 3> s_t;

    mutable etl::dyn_matrix<float, 3> delta_t;
    mutable etl::dyn_matrix<float, 3> d_h_t;
    mutable etl::dyn_matrix<float, 3> d_x_t;

    void prepare_cache(size_t Batch, size_t time_steps, size_t sequence_length, size_t hidden_units) const {
        if (cpp_unlikely(!x_t.memory_start())) {
            // Forward cache
            x_t.resize(time_steps, Batch, sequence_length);
            s_t.resize(time_steps, Batch, hidden_units);

            // Backward cache
            delta_t.resize(time_steps, Batch, hidden_units);
            d_h_t.resize(time_steps, Batch, hidden_units);
            d_x_t.resize(time_steps, Batch, sequence_length);
        }
    }

    /*!
     * \brief Apply the layer to the given batch of input.
     *
     * \param x A batch of input
     * \param output A batch of output that will be filled
     * \param w The W weights matrix
     * \param u The U weights matrix
     */
    template <typename H, typename V, typename W, typename U, typename B>
    void forward_batch_impl(H&& output, const V& x, const W& w, const U& u, const B& b, size_t time_steps, size_t sequence_length, size_t hidden_units) const {
        const auto Batch = etl::dim<0>(x);

        prepare_cache(Batch, time_steps, sequence_length, hidden_units);

        // 1. Rearrange input
        x_t = transpose_front(x);

        // 2. Forward propagation through time

        // t == 0

        s_t(0) = f_activate<activation_function>(bias_add_2d(x_t(0) * u, b));

        for (size_t t = 1; t < time_steps; ++t) {
            s_t(t) = f_activate<activation_function>(bias_add_2d(x_t(t) * u + s_t(t - 1) * w, b));
        }

        // 3. Rearrange the output<commit_msg>Improve GPU performance of RNN<commit_after>//=======================================================================
// Copyright (c) 2014-2020 <NAME>
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include <fstream>

#include "cpp_utils/assert.hpp" //Assertions
#include "cpp_utils/io.hpp"     // For binary writing

#include "etl/etl.hpp"

#include "layer.hpp"
#include "layer_traits.hpp"
#include "util/tmp.hpp"

namespace dll {

/*!
 * \brief Base class for RNN layers (fast / dynamic)
 */
template <typename Derived, typename Desc>
struct base_rnn_layer : layer<Derived> {
    using desc      = Desc;                            ///< The descriptor of the layer
    using derived_t = Derived;                         ///< The derived type (CRTP)
    using weight    = typename desc::weight;           ///< The data type for this layer
    using this_type = base_rnn_layer<derived_t, desc>; ///< The type of this layer
    using base_type = layer<Derived>;                  ///< The base type

    static constexpr auto activation_function = desc::activation_function; ///< The layer's activation function

    /*!
     * \brief Initialize the neural layer
     */
    base_rnn_layer()
            : base_type() {
        // Nothing to init here
    }

    base_rnn_layer(const base_rnn_layer& rhs) = delete;
    base_rnn_layer(base_rnn_layer&& rhs)      = delete;

    base_rnn_layer& operator=(const base_rnn_layer& rhs) = delete;
    base_rnn_layer& operator=(base_rnn_layer&& rhs) = delete;

    mutable etl::dyn_matrix<float, 3> x_t;
    mutable etl::dyn_matrix<float, 3> s_t;

    mutable etl::dyn_matrix<float, 3> delta_t;
    mutable etl::dyn_matrix<float, 3> d_h_t;
    mutable etl::dyn_matrix<float, 3> d_x_t;

    void prepare_cache(size_t Batch, size_t time_steps, size_t sequence_length, size_t hidden_units) const {
        if (cpp_unlikely(!x_t.memory_start())) {
            // Forward cache
            x_t.resize(time_steps, Batch, sequence_length);
            s_t.resize(time_steps, Batch, hidden_units);

            // Backward cache
            delta_t.resize(time_steps, Batch, hidden_units);
            d_h_t.resize(time_steps, Batch, hidden_units);
            d_x_t.resize(time_steps, Batch, sequence_length);

#ifdef ETL_GPU
            // Note: all these matrices are only accessed through sub views
            // So, the base CPU/GPU is never fully updated.
            // Starting with valid GPU status ensures that we never copy between
            // CPU and GPU
            d_h_t.ensure_gpu_up_to_date();
            s_t.ensure_gpu_up_to_date();
            d_h_t.ensure_gpu_up_to_date();
            d_x_t.ensure_gpu_up_to_date();
#endif
        }
    }

    /*!
     * \brief Apply the layer to the given batch of input.
     *
     * \param x A batch of input
     * \param output A batch of output that will be filled
     * \param w The W weights matrix
     * \param u The U weights matrix
     */
    template <typename H, typename V, typename W, typename U, typename B>
    void forward_batch_impl(H&& output, const V& x, const W& w, const U& u, const B& b, size_t time_steps, size_t sequence_length, size_t hidden_units) const {
        const auto Batch = etl::dim<0>(x);

        prepare_cache(Batch, time_steps, sequence_length, hidden_units);

        // 1. Rearrange input
        x_t = transpose_front(x);

        // 2. Forward propagation through time

        // t == 0

        s_t(0) = f_activate<activation_function>(bias_add_2d(x_t(0) * u, b));

        for (size_t t = 1; t < time_steps; ++t) {
            s_t(t) = f_activate<activation_function>(bias_add_2d(x_t(t) * u + s_t(t - 1) * w, b));
        }

        // 3. Rearrange the output
{'fraction_non_alphanumeric': 0.14348302300109528, 'fraction_numerical': 0.0065717415115005475, 'mean_word_length': 2.6289120715350225, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '219757', 'n_tokens_mistral': 2374, 'n_tokens_neox': 2246, 'n_words': 640}