<commit_before>#ifndef NN_RECURRENT_H
#define NN_RECURRENT_H

#include "container.h"
#include "sequential.h"
#include "linear.h"
#include "tanh.h"

namespace nnlib
{

/// Vanilla recurrent module.
/// output(t) = squash(feedback(output(t - 1)) + squash(input(t)))
template <typename T = double>
class Recurrent : public Container<T>
{
public:
	using Container<T>::add;
	using Container<T>::inputs;
	using Container<T>::outputs;
	using Container<T>::batch;
	
	Recurrent(size_t inps, size_t outs, size_t bats = 1) :
		m_inputModule(new Linear<T>(inps, outs, bats)),
		m_feedbackModule(new Linear<T>(outs, outs, bats)),
		m_outputModule(new Sequential<T>(new Linear<T>(outs, outs, bats), new TanH<>())),
		m_state(bats, outs),
		m_stateGrad(bats, outs),
		m_prevState(bats, outs),
		m_resetStateGrad(true)
	{
		add(m_inputModule, m_feedbackModule, m_outputModule);
		m_state.fill(0);
	}
	
	Recurrent(Module<T> *inputModule, Module<T> *feedbackModule, Module<T> *outputModule) :
		m_inputModule(inputModule),
		m_feedbackModule(feedbackModule),
		m_outputModule(outputModule),
		m_state(m_outputModule->outputs(), true),
		m_stateGrad(m_outputModule->outputs(), true),
		m_prevState(m_outputModule->outputs(), true),
		m_resetStateGrad(true)
	{
		add(m_inputModule, m_feedbackModule, m_outputModule);
		m_state.fill(0);
	}
	
	Recurrent &reset()
	{
		m_state.fill(0);
		return *this;
	}
	
	// MARK: Module methods
	
	/// Forward propagate input, returning output.
	virtual Tensor<T> &forward(const Tensor<T> &input) override
	{
		m_prevState.copy(m_state);
		m_inputModule->forward(input);
		m_feedbackModule->forward(m_prevState);
		m_state.copy(m_inputModule->output()).addMM(m_feedbackModule->output());
		m_resetStateGrad = true;
		return m_outputModule->forward(m_state);
	}
	
	/// Backward propagate input and output gradient, returning input gradient.
	virtual Tensor<T> &backward(const Tensor<T> &input, const Tensor<T> &outGrad) override
	{
		if(m_resetStateGrad)
		{
			m_resetStateGrad = false;
			m_stateGrad.fill(0);
		}
		m_outputModule->backward(m_state, outGrad);
		m_outputModule->inGrad().addMM(m_stateGrad);
		m_stateGrad.copy(m_feedbackModule->backward(m_prevState, m_outputModule->inGrad()));
		m_inputModule->backward(input, m_outputModule->inGrad());
		return m_inputModule->inGrad();
	}
	
	/// Cached output.<commit_msg>Added runtime error to misuse of recurrent.<commit_after>#ifndef NN_RECURRENT_H
#define NN_RECURRENT_H

#include "container.h"
#include "sequential.h"
#include "linear.h"
#include "tanh.h"

namespace nnlib
{

/// Vanilla recurrent module.
/// output(t) = squash(feedback(output(t - 1)) + squash(input(t)))
template <typename T = double>
class Recurrent : public Container<T>
{
public:
	using Container<T>::inputs;
	using Container<T>::outputs;
	using Container<T>::batch;
	
	Recurrent(size_t inps, size_t outs, size_t bats = 1) :
		m_inputModule(new Linear<T>(inps, outs, bats)),
		m_feedbackModule(new Linear<T>(outs, outs, bats)),
		m_outputModule(new Sequential<T>(new Linear<T>(outs, outs, bats), new TanH<>())),
		m_state(bats, outs),
		m_stateGrad(bats, outs),
		m_prevState(bats, outs),
		m_resetStateGrad(true)
	{
		Container<T>::add(m_inputModule);
		Container<T>::add(m_feedbackModule);
		Container<T>::add(m_outputModule);
		m_state.fill(0);
	}
	
	Recurrent(Module<T> *inputModule, Module<T> *feedbackModule, Module<T> *outputModule) :
		m_inputModule(inputModule),
		m_feedbackModule(feedbackModule),
		m_outputModule(outputModule),
		m_state(m_outputModule->outputs(), true),
		m_stateGrad(m_outputModule->outputs(), true),
		m_prevState(m_outputModule->outputs(), true),
		m_resetStateGrad(true)
	{
		Container<T>::add(m_inputModule);
		Container<T>::add(m_feedbackModule);
		Container<T>::add(m_outputModule);
		m_state.fill(0);
	}
	
	Recurrent &reset()
	{
		m_state.fill(0);
		return *this;
	}
	
	// MARK: Container methods
	
	/// Cannot add a component to this container.
	virtual void add(Module<T> *component) override
	{
		throw std::runtime_error("Cannot add components to a recurrent module!");
	}
	
	// MARK: Module methods
	
	/// Forward propagate input, returning output.
	virtual Tensor<T> &forward(const Tensor<T> &input) override
	{
		m_prevState.copy(m_state);
		m_inputModule->forward(input);
		m_feedbackModule->forward(m_prevState);
		m_state.copy(m_inputModule->output()).addMM(m_feedbackModule->output());
		m_resetStateGrad = true;
		return m_outputModule->forward(m_state);
	}
	
	/// Backward propagate input and output gradient, returning input gradient.
	virtual Tensor<T> &backward(const Tensor<T> &input, const Tensor<T> &outGrad) override
	{
		if(m_resetStateGrad)
		{
			m_resetStateGrad = false;
			m_stateGrad.fill(0);
		}
		m_outputModule->backward(m_state, outGrad);
		m_outputModule->inGrad().addMM(m_stateGrad);
		m_stateGrad.copy(m_feedbackModule->backward(m_prevState, m_outputModule->inGrad()));
		m_inputModule->backward(input, m_outputModule->inGrad());
		return m_inputModule->inGrad();
	}
	
	/// Cached output.
{'fraction_non_alphanumeric': 0.14548361310951238, 'fraction_numerical': 0.002398081534772182, 'mean_word_length': 5.718120805369128, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7193160', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1749, 'n_words': 329}