<commit_before>#pragma once

#include "layer.h"
#include "tensor/numeric.h"

namespace nano
{
        ///
        /// \brief activation layer: applies a non-linear scalar function to the each scalar input.
        ///
        template <typename top>
        class activation_layer_t final : public layer_t
        {
        public:

                rlayer_t clone() const final;
                json_reader_t& config(json_reader_t& reader) final { return reader; }
                json_writer_t& config(json_writer_t& writer) const final { return writer; }

                bool resize(const tensor3d_dims_t& idims) final;

                void random(vector_map_t pdata) const final;
                void output(tensor4d_cmaps_t idata, vector_cmap_t pdata, tensor4d_map_t odata) final;
                void ginput(tensor4d_maps_t idata, vector_cmap_t pdata, tensor4d_cmap_t odata) final;
                void gparam(tensor4d_cmaps_t idata, vector_map_t pdata, tensor4d_cmap_t odata) final;

                tensor_size_t psize() const final { return 0; }
                tensor3d_dim_t odims() const final { return m_xdims; }
                tensor_size_t flops_output() const final { return 10 * nano::size(odims()); }
                tensor_size_t flops_ginput() const final { return 10 * nano::size(odims()); }
                tensor_size_t flops_gparam() const final { return 0; }

        private:

                auto xsize() const { return nano::size(odims()); }

                // attributes
                tensor3d_dim_t m_xdims{{0, 0, 0}};     ///< input/output dimensions
        };

        template <typename top>
        rlayer_t activation_layer_t<top>::clone() const
        {
                return std::make_unique<activation_layer_t<top>>(*this);
        }

        template <typename top>
        bool activation_layer_t<top>::resize(const tensor3d_dims_t& idims)
        {
                if (idims.size() != 1)
                {
                        return false;
                }

                m_xdims = idims[0];
                return true;
        }

        template <typename top>
        void activation_layer_t<top>::random(vector_map_t pdata) const
        {
                assert(pdata.size() == psize());
                NANO_UNUSED1_RELEASE(pdata);
        }

        template <typename top>
        void activation_layer_t<top>::output(tensor4d_cmaps_t idata, vector_cmap_t pdata, tensor4d_map_t odata)
        {
                assert(idata.size() == 1);
                assert(idata[0].dims() == odata.dims());
                assert(pdata.size() == psize());
                assert(nano::isfinite(idata[0]));

                top::output(idata[0].array(), odata.array());

                assert(nano::isfinite(odata));
                NANO_UNUSED1_RELEASE(pdata);
        }

        template <typename top>
        void activation_layer_t<top>::ginput(tensor4d_maps_t idata, vector_cmap_t pdata, tensor4d_cmap_t odata)
        {
                assert(idata.size() == 1);
                assert(idata[0].dims() == odata.dims());
                assert(pdata.size() == psize());
                assert(nano::isfinite(idata[0]));
                assert(nano::isfinite(odata));

                top::ginput(idata[0].array(), odata.array());

                assert(nano::isfinite(idata[0]));
                NANO_UNUSED1_RELEASE(pdata);
        }

        template <typename top>
        void activation_layer_t<top>::gparam(tensor4d_cmaps_t idata, vector_map_t pdata, tensor4d_cmap_t odata)
        {
                assert(idata.size() == 1);
                assert(idata[0].dims() == odata.dims());
                assert(pdata.size() == psize());
                NANO_UNUSED3_RELEASE(idata, pdata, odata);
        }

        ///
        /// \brief sin(x) activation function.
        ///
        struct activation_sine_t
        {
                template <typename tiarray, typename toarray>
                static void output(const tiarray& idata, toarray&& odata)
                {
                        odata = idata.sin();
                }

                template <typename tiarray, typename toarray>
                static void ginput(tiarray&& idata, const toarray& odata)
                {
                        idata = odata * idata.cos();
                }
        };

        using activation_layer_sine_t = activation_layer_t<activation_sine_t>;

        ///
        /// \brief x/sqrt(1+x^2) activation function.
        ///
        struct activation_snorm_t
        {
                template <typename tiarray, typename toarray>
                static void output(const tiarray& idata, toarray&& odata)
                {
                        odata = idata / (1 + idata.square()).sqrt();
                }

                template <typename tiarray, typename toarray>
                static void ginput(tiarray&& idata, const toarray& odata)
                {
                        idata = odata / (1 + idata.square()).cube().sqrt();
                }
        };

        using activation_layer_snorm_t = activation_layer_t<activation_snorm_t>;

        ///
        /// \brief (e^x-e^-x)/(e^x+e^-x) hyperbolic tangent activation function.
        ///
        struct activation_tanh_t
        {
                template <typename tiarray, typename toarray>
                static void output(const tiarray& idata, toarray&& odata)
                {
                        odata = idata.tanh();
                }<commit_msg>use Eigen's rsqrt (faster than 1/sqrt)<commit_after>#pragma once

#include "layer.h"
#include "tensor/numeric.h"

namespace nano
{
        ///
        /// \brief activation layer: applies a non-linear scalar function to the each scalar input.
        ///
        template <typename top>
        class activation_layer_t final : public layer_t
        {
        public:

                rlayer_t clone() const final;
                json_reader_t& config(json_reader_t& reader) final { return reader; }
                json_writer_t& config(json_writer_t& writer) const final { return writer; }

                bool resize(const tensor3d_dims_t& idims) final;

                void random(vector_map_t pdata) const final;
                void output(tensor4d_cmaps_t idata, vector_cmap_t pdata, tensor4d_map_t odata) final;
                void ginput(tensor4d_maps_t idata, vector_cmap_t pdata, tensor4d_cmap_t odata) final;
                void gparam(tensor4d_cmaps_t idata, vector_map_t pdata, tensor4d_cmap_t odata) final;

                tensor_size_t psize() const final { return 0; }
                tensor3d_dim_t odims() const final { return m_xdims; }
                tensor_size_t flops_output() const final { return 10 * nano::size(odims()); }
                tensor_size_t flops_ginput() const final { return 10 * nano::size(odims()); }
                tensor_size_t flops_gparam() const final { return 0; }

        private:

                auto xsize() const { return nano::size(odims()); }

                // attributes
                tensor3d_dim_t m_xdims{{0, 0, 0}};     ///< input/output dimensions
        };

        template <typename top>
        rlayer_t activation_layer_t<top>::clone() const
        {
                return std::make_unique<activation_layer_t<top>>(*this);
        }

        template <typename top>
        bool activation_layer_t<top>::resize(const tensor3d_dims_t& idims)
        {
                if (idims.size() != 1)
                {
                        return false;
                }

                m_xdims = idims[0];
                return true;
        }

        template <typename top>
        void activation_layer_t<top>::random(vector_map_t pdata) const
        {
                assert(pdata.size() == psize());
                NANO_UNUSED1_RELEASE(pdata);
        }

        template <typename top>
        void activation_layer_t<top>::output(tensor4d_cmaps_t idata, vector_cmap_t pdata, tensor4d_map_t odata)
        {
                assert(idata.size() == 1);
                assert(idata[0].dims() == odata.dims());
                assert(pdata.size() == psize());
                assert(nano::isfinite(idata[0]));

                top::output(idata[0].array(), odata.array());

                assert(nano::isfinite(odata));
                NANO_UNUSED1_RELEASE(pdata);
        }

        template <typename top>
        void activation_layer_t<top>::ginput(tensor4d_maps_t idata, vector_cmap_t pdata, tensor4d_cmap_t odata)
        {
                assert(idata.size() == 1);
                assert(idata[0].dims() == odata.dims());
                assert(pdata.size() == psize());
                assert(nano::isfinite(idata[0]));
                assert(nano::isfinite(odata));

                top::ginput(idata[0].array(), odata.array());

                assert(nano::isfinite(idata[0]));
                NANO_UNUSED1_RELEASE(pdata);
        }

        template <typename top>
        void activation_layer_t<top>::gparam(tensor4d_cmaps_t idata, vector_map_t pdata, tensor4d_cmap_t odata)
        {
                assert(idata.size() == 1);
                assert(idata[0].dims() == odata.dims());
                assert(pdata.size() == psize());
                NANO_UNUSED3_RELEASE(idata, pdata, odata);
        }

        ///
        /// \brief sin(x) activation function.
        ///
        struct activation_sine_t
        {
                template <typename tiarray, typename toarray>
                static void output(const tiarray& idata, toarray&& odata)
                {
                        odata = idata.sin();
                }

                template <typename tiarray, typename toarray>
                static void ginput(tiarray&& idata, const toarray& odata)
                {
                        idata = odata * idata.cos();
                }
        };

        using activation_layer_sine_t = activation_layer_t<activation_sine_t>;

        ///
        /// \brief x/sqrt(1+x^2) activation function.
        ///
        struct activation_snorm_t
        {
                template <typename tiarray, typename toarray>
                static void output(const tiarray& idata, toarray&& odata)
                {
                        odata = idata * (1 + idata.square()).rsqrt();
                }

                template <typename tiarray, typename toarray>
                static void ginput(tiarray&& idata, const toarray& odata)
                {
                        idata = odata * (1 + idata.square()).cube().rsqrt();
                }
        };

        using activation_layer_snorm_t = activation_layer_t<activation_snorm_t>;

        ///
        /// \brief (e^x-e^-x)/(e^x+e^-x) hyperbolic tangent activation function.
        ///
        struct activation_tanh_t
        {
                template <typename tiarray, typename toarray>
                static void output(const tiarray& idata, toarray&& odata)
                {
                        odata = idata.tanh();
                }
{'fraction_non_alphanumeric': 0.1059026203644936, 'fraction_numerical': 0.00843231480641944, 'mean_word_length': 1.6268159085496547, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '900700', 'n_tokens_mistral': 3334, 'n_tokens_neox': 3107, 'n_words': 651}