<commit_before>#ifndef MLEARN_FULLY_CONNECTED_NET_EXPLORER_INCLUDE
#define MLEARN_FULLY_CONNECTED_NET_EXPLORER_INCLUDE

// MLearn includes
#include <MLearn/Core>
#include "../../ActivationFunction.h"
#include "../../CommonImpl.h"

// STL
#include <random>
#include <chrono>

namespace MLearn{

	namespace NeuralNets{

		namespace FeedForwardNets{

			/*!
			*	\brief		Helper class for performing dropout
			*	\author 	phineasng
			*
			*/
			template < bool DROPOUT = false >
			struct UnitProcessor{
				template < 	ActivationType TYPE,
							typename WeightType,
							typename RNG,
							typename DERIVED,
							typename DERIVED_BOOL >
				static inline void process( Eigen::VectorBlock<DERIVED> pre_activations, Eigen::VectorBlock<DERIVED> activations, Eigen::VectorBlock<DERIVED_BOOL> dropped, RNG& rng, std::bernoulli_distribution& b_dist, const WeightType& inv_p  ){
					static_assert( std::is_floating_point<typename DERIVED::Scalar>::value,"The scalar type has to be floating point!");
					static_assert( std::is_floating_point<WeightType>::value,"The declared scalar type and the scalar type of the input vector have to be the same!");
					activations = pre_activations.unaryExpr(std::pointer_to_unary_function<WeightType,WeightType>(ActivationFunction<TYPE>::evaluate));
				}
				template < 	typename WeightType,
							typename DERIVED_BOOL >
				static inline void process_derivatives( Eigen::Ref< MLVector<WeightType> > derivatives ,Eigen::VectorBlock<DERIVED_BOOL> dropped){static_assert( std::is_floating_point<WeightType>::value,"The scalar type has to be floating point!");}
			};
			template <>
			struct UnitProcessor<true>{
				template < 	ActivationType TYPE,
							typename WeightType,
							typename RNG,
							typename DERIVED,
							typename DERIVED_BOOL >
				static inline void process( Eigen::VectorBlock<DERIVED> pre_activations, Eigen::VectorBlock<DERIVED> activations, Eigen::VectorBlock<DERIVED_BOOL> dropped, RNG& rng, std::bernoulli_distribution& b_dist, const WeightType& inv_p ){
					static_assert( std::is_floating_point<typename DERIVED::Scalar>::value,"The scalar type has to be floating point!");
					static_assert( std::is_floating_point<WeightType>::value,"The declared scalar type and the scalar type of the input vector have to be the same!");
					for ( decltype(activations.size()) idx = 0; idx < activations.size(); ++idx ){
						if ( b_dist(rng) ){
							activations[idx] = ActivationFunction<TYPE>::evaluate(pre_activations[idx])*inv_p;
							dropped[idx] = false;
						}else{
							activations[idx] = WeightType(0);
							pre_activations[idx] = WeightType(0);
							dropped[idx] = true;
						}
					}
				}
				template < 	typename WeightType,
							typename DERIVED_BOOL >
				static inline void process_derivatives( Eigen::Ref< MLVector<WeightType> > derivatives ,Eigen::VectorBlock<DERIVED_BOOL> dropped){
					static_assert( std::is_floating_point<WeightType>::value,"The scalar type has to be floating point!");
					for ( decltype(derivatives.size()) idx = 0; idx < derivatives.size(); ++idx ){
						if ( dropped[idx] ){
							derivatives[idx] = WeightType(0);
						}
					}
				}
			};

			/*! 
			*	\brief		Supporting class to perform the common feedforward fully connected neural nets
			*	\details  	Class implementing the feedforward pass and the backpropagation algorithm for 
			*				feedforward fully connected neural nets.
			*				The weights (and biases) are considered to be stored in a vector, in a 
			*				row major way and sequentially from input to output 
			*				An array of integers is used to describe the layers and interpret the weights vector.
			*	\author 	phineasng
			*/
			template < 	typename WeightType, 
						typename IndexType,
						ActivationType HiddenLayerActivation = ActivationType::LOGISTIC,
						ActivationType OutputLayerActivation = ActivationType::LINEAR >
			class FCNetsExplorer{
			public:
				static_assert( std::is_floating_point<WeightType>::value, "The weights type has to be floating point!" );
				static_assert( std::is_integral<IndexType>::value && std::is_unsigned<IndexType>::value, "The index type has to be integer and unsigned!" );
				// TYPEDEFS
				typedef WeightType 				Scalar_T;
				typedef IndexType 				Index_T;
				static const ActivationType 	HiddenActivationType = HiddenLayerActivation;
				static const ActivationType 	OutputActivationType = OutputLayerActivation;
				// CONSTRUCTOR
				FCNetsExplorer(const MLVector< IndexType >& refLayers):
					layers(refLayers),
					activations( refLayers.array().sum() ),
					pre_activations( refLayers.array().sum() ),
					dropped( refLayers.array().sum() ),
					derivatives( refLayers.maxCoeff() ),
					rng(static_cast<std::mt19937::result_type>(std::chrono::system_clock::now().time_since_epoch().count())),
					b_dist(0.5) {
						MLEARN_ASSERT( layers.size() > 1, "Not a deep architecture: at least 2 layers are needed for the algorithm to work!" );
						MLEARN_WARNING( ( layers.array() > 0 ).all(), "Some layers have zero units!" );
						inv_p = WeightType(1)/b_dist.p();
					}
				// FORWARD PASS
				template< 	bool DROPOUT = false,
							typename DERIVED,
							typename DERIVED_2 >
				void forwardpass( const Eigen::MatrixBase< DERIVED >& weights, const Eigen::MatrixBase< DERIVED_2 >& input ){
					static_assert( std::is_same<typename DERIVED::Scalar,typename DERIVED_2::Scalar >::value, "The vectors given as input has to have same scalar type!" );
					static_assert( std::is_same<typename DERIVED::Scalar,WeightType >::value, "The vectors' scalar type has to be consistent to the declared weight type!" );
					static_assert( (DERIVED::ColsAtCompileTime == 1) && (DERIVED_2::ColsAtCompileTime == 1), "The inputs have to be column vectors (or consistent structures)!");
					MLEARN_ASSERT( ( layers.head(layers.size()-1).dot(layers.tail(layers.size()-1)) + layers.tail(layers.size()-1).array().sum()) == weights.size(), "Number of weights is not consistent with number of units in the layers!" );
					MLEARN_ASSERT( input.size() == layers[0] , "Input should be of the declared size of the first layer!" );
					
					pre_activations.segment(0,layers[0]) = input;
					UnitProcessor<DROPOUT>::template process<ActivationType::LINEAR,WeightType,RNG_TYPE>(pre_activations.segment(0,layers[0]),activations.segment(0,layers[0]),dropped.segment(0,layers[0]),rng,b_dist,inv_p);
					
					// declare refs to derived: if DERIVED_* are expressions then they will be evaluated otherwise we can access its data without copies
					const Eigen::Ref<const Eigen::Matrix< typename DERIVED::Scalar, DERIVED::RowsAtCompileTime, DERIVED::ColsAtCompileTime > >& ref_weights(weights);
					
					decltype(activations.size()) offset = layers[0];
					decltype(weights.size()) offset_weights = 0;
					decltype(weights.size()) tmp;
					decltype(activations.size()) idx;
					decltype(idx) idx_m1;

					for ( idx = 1; idx < (layers.size() - 1); ++idx ){

						idx_m1 = idx - 1;
						tmp = layers[idx_m1]*layers[idx];
						pre_activations.segment(offset,layers[idx]) = Eigen::Map<const MLMatrix<WeightType>>( ref_weights.data() + offset_weights, layers[idx], layers[idx_m1] )*activations.segment(offset-layers[idx_m1],layers[idx_m1]);
						offset_weights += tmp;
						pre_activations.segment(offset,layers[idx]) += ref_weights.segment(offset_weights,layers[idx]);	
						UnitProcessor<DROPOUT>::template process<HiddenLayerActivation,WeightType,RNG_TYPE>(pre_activations.segment(offset,layers[idx]),activations.segment(offset,layers[idx]),dropped.segment(offset,layers[idx]),rng,b_dist,inv_p);
						offset_weights += layers[idx];
						offset += layers[idx];  
					}
					
					idx_m1 = idx - 1;
					tmp = layers[idx_m1]*layers[idx];
					pre_activations.segment(offset,layers[idx]) = Eigen::Map<const MLMatrix<WeightType>>( ref_weights.data() + offset_weights, layers[idx], layers[idx_m1] )*activations.segment(offset-layers[idx_m1],layers[idx_m1]);
					offset_weights += tmp;
					pre_activations.segment(offset,layers[idx]) += weights.segment(offset_weights,layers[idx]);	
					activations.segment(offset,layers[idx]) = pre_activations.segment(offset,layers[idx]).unaryExpr(std::pointer_to_unary_function<WeightType,WeightType>(ActivationFunction<OutputLayerActivation>::evaluate));
					
				}
				// BACKPROPAGATION	
				/*!
				*	\brief 		Backpropagation algorithm
				*	\details	It assumes that a forward pass has already been performed.
				*/
				template< 	bool DROPOUT = false, 
							typename DERIVED,
							typename DERIVED_2,
							typename DERIVED_3 >
				void backpropagate( const Eigen::MatrixBase<DERIVED>& weights, const Eigen::MatrixBase<DERIVED_2>& gradient_output, Eigen::MatrixBase<DERIVED_3>& gradient_weights ){
					static_assert( std::is_same<typename DERIVED::Scalar,typename DERIVED_2::Scalar >::value && std::is_same<typename DERIVED::Scalar,typename DERIVED_3::Scalar >::value, "The vectors given as input has to have same scalar type!" );
					static_assert( std::is_same<typename DERIVED::Scalar,WeightType >::value, "The vectors' scalar type has to be consistent to the declared weight type!" );
					static_assert( (DERIVED::ColsAtCompileTime == 1) && (DERIVED_2::ColsAtCompileTime == 1) && (DERIVED_3::ColsAtCompileTime == 1), "The inputs have to be column vectors (or consistent structures)!");
					MLEARN_ASSERT( ( layers.head(layers.size()-1).dot(layers.tail(layers.size()-1)) + layers.tail(layers.size()-1).array().sum()) == weights.size(), "Number of weights is not consistent with number of units in the layers!" );
					MLEARN_ASSERT( gradient_output.size() == layers[layers.size()-1], "Output layer not consistent with the given gradient!" );
					MLEARN_ASSERT( gradient_weights.size() == weights.size(), "Weights size and respective gradient size not compatible!" );

					// 
					const DERIVED& ref_weights = weights.derived();
					
					decltype(activations.size()) idx = layers.size()-1;
					decltype(activations.size()) idx_m1 = idx-1;
					decltype(activations.size()) offset = layers.head(idx).array().sum();
					decltype(ref_weights.size()) offset_weights = ref_weights.size() - layers[idx];
					decltype(ref_weights.size()) tmp = layers[idx_m1]*layers[idx];

					// compute delta output layer and assign it to the gradientof the last biases
					InternalImpl::DerivativeWrapper<OutputLayerActivation>::template derive<WeightType>( pre_activations.segment(offset,layers[idx]),activations.segment(offset,layers[idx]),derivatives.head(layers[idx]) );
					gradient_weights.segment(offset_weights,layers[idx]) = gradient_output.cwiseProduct(derivatives.head(layers[idx]));<commit_msg>Changed constness of forwardpass routine<commit_after>#ifndef MLEARN_FULLY_CONNECTED_NET_EXPLORER_INCLUDE
#define MLEARN_FULLY_CONNECTED_NET_EXPLORER_INCLUDE

// MLearn includes
#include <MLearn/Core>
#include "../../ActivationFunction.h"
#include "../../CommonImpl.h"

// STL
#include <random>
#include <chrono>

namespace MLearn{

	namespace NeuralNets{

		namespace FeedForwardNets{

			/*!
			*	\brief		Helper class for performing dropout
			*	\author 	phineasng
			*
			*/
			template < bool DROPOUT = false >
			struct UnitProcessor{
				template < 	ActivationType TYPE,
							typename WeightType,
							typename RNG,
							typename DERIVED,
							typename DERIVED_BOOL >
				static inline void process( Eigen::VectorBlock<DERIVED> pre_activations, Eigen::VectorBlock<DERIVED> activations, Eigen::VectorBlock<DERIVED_BOOL> dropped, RNG& rng, std::bernoulli_distribution& b_dist, const WeightType& inv_p  ){
					static_assert( std::is_floating_point<typename DERIVED::Scalar>::value,"The scalar type has to be floating point!");
					static_assert( std::is_floating_point<WeightType>::value,"The declared scalar type and the scalar type of the input vector have to be the same!");
					activations = pre_activations.unaryExpr(std::pointer_to_unary_function<WeightType,WeightType>(ActivationFunction<TYPE>::evaluate));
				}
				template < 	typename WeightType,
							typename DERIVED_BOOL >
				static inline void process_derivatives( Eigen::Ref< MLVector<WeightType> > derivatives ,Eigen::VectorBlock<DERIVED_BOOL> dropped){static_assert( std::is_floating_point<WeightType>::value,"The scalar type has to be floating point!");}
			};
			template <>
			struct UnitProcessor<true>{
				template < 	ActivationType TYPE,
							typename WeightType,
							typename RNG,
							typename DERIVED,
							typename DERIVED_BOOL >
				static inline void process( Eigen::VectorBlock<DERIVED> pre_activations, Eigen::VectorBlock<DERIVED> activations, Eigen::VectorBlock<DERIVED_BOOL> dropped, RNG& rng, std::bernoulli_distribution& b_dist, const WeightType& inv_p ){
					static_assert( std::is_floating_point<typename DERIVED::Scalar>::value,"The scalar type has to be floating point!");
					static_assert( std::is_floating_point<WeightType>::value,"The declared scalar type and the scalar type of the input vector have to be the same!");
					for ( decltype(activations.size()) idx = 0; idx < activations.size(); ++idx ){
						if ( b_dist(rng) ){
							activations[idx] = ActivationFunction<TYPE>::evaluate(pre_activations[idx])*inv_p;
							dropped[idx] = false;
						}else{
							activations[idx] = WeightType(0);
							pre_activations[idx] = WeightType(0);
							dropped[idx] = true;
						}
					}
				}
				template < 	typename WeightType,
							typename DERIVED_BOOL >
				static inline void process_derivatives( Eigen::Ref< MLVector<WeightType> > derivatives ,Eigen::VectorBlock<DERIVED_BOOL> dropped){
					static_assert( std::is_floating_point<WeightType>::value,"The scalar type has to be floating point!");
					for ( decltype(derivatives.size()) idx = 0; idx < derivatives.size(); ++idx ){
						if ( dropped[idx] ){
							derivatives[idx] = WeightType(0);
						}
					}
				}
			};

			/*! 
			*	\brief		Supporting class to perform the common feedforward fully connected neural nets
			*	\details  	Class implementing the feedforward pass and the backpropagation algorithm for 
			*				feedforward fully connected neural nets.
			*				The weights (and biases) are considered to be stored in a vector, in a 
			*				row major way and sequentially from input to output 
			*				An array of integers is used to describe the layers and interpret the weights vector.
			*	\author 	phineasng
			*/
			template < 	typename WeightType, 
						typename IndexType,
						ActivationType HiddenLayerActivation = ActivationType::LOGISTIC,
						ActivationType OutputLayerActivation = ActivationType::LINEAR >
			class FCNetsExplorer{
			public:
				static_assert( std::is_floating_point<WeightType>::value, "The weights type has to be floating point!" );
				static_assert( std::is_integral<IndexType>::value && std::is_unsigned<IndexType>::value, "The index type has to be integer and unsigned!" );
				// TYPEDEFS
				typedef WeightType 				Scalar_T;
				typedef IndexType 				Index_T;
				static const ActivationType 	HiddenActivationType = HiddenLayerActivation;
				static const ActivationType 	OutputActivationType = OutputLayerActivation;
				// CONSTRUCTOR
				FCNetsExplorer(const MLVector< IndexType >& refLayers):
					layers(refLayers),
					activations( refLayers.array().sum() ),
					pre_activations( refLayers.array().sum() ),
					dropped( refLayers.array().sum() ),
					derivatives( refLayers.maxCoeff() ),
					rng(static_cast<std::mt19937::result_type>(std::chrono::system_clock::now().time_since_epoch().count())),
					b_dist(0.5) {
						MLEARN_ASSERT( layers.size() > 1, "Not a deep architecture: at least 2 layers are needed for the algorithm to work!" );
						MLEARN_WARNING( ( layers.array() > 0 ).all(), "Some layers have zero units!" );
						inv_p = WeightType(1)/b_dist.p();
					}
				// FORWARD PASS
				template< 	bool DROPOUT = false,
							typename DERIVED,
							typename DERIVED_2 >
				void forwardpass( const Eigen::MatrixBase< DERIVED >& weights, const Eigen::MatrixBase< DERIVED_2 >& input ){
					static_assert( std::is_same<typename DERIVED::Scalar,typename DERIVED_2::Scalar >::value, "The vectors given as input has to have same scalar type!" );
					static_assert( std::is_same<typename DERIVED::Scalar,WeightType >::value, "The vectors' scalar type has to be consistent to the declared weight type!" );
					static_assert( (DERIVED::ColsAtCompileTime == 1) && (DERIVED_2::ColsAtCompileTime == 1), "The inputs have to be column vectors (or consistent structures)!");
					MLEARN_ASSERT( ( layers.head(layers.size()-1).dot(layers.tail(layers.size()-1)) + layers.tail(layers.size()-1).array().sum()) == weights.size(), "Number of weights is not consistent with number of units in the layers!" );
					MLEARN_ASSERT( input.size() == layers[0] , "Input should be of the declared size of the first layer!" );
					
					pre_activations.segment(0,layers[0]) = input;
					UnitProcessor<DROPOUT>::template process<ActivationType::LINEAR,WeightType,RNG_TYPE>(pre_activations.segment(0,layers[0]),activations.segment(0,layers[0]),dropped.segment(0,layers[0]),rng,b_dist,inv_p);
					
					// declare refs to derived: if DERIVED_* are expressions then they will be evaluated otherwise we can access its data without copies
					const Eigen::Ref<const Eigen::Matrix< typename DERIVED::Scalar, DERIVED::RowsAtCompileTime, DERIVED::ColsAtCompileTime > >& ref_weights(weights);
					
					decltype(activations.size()) offset = layers[0];
					decltype(weights.size()) offset_weights = 0;
					decltype(weights.size()) tmp;
					decltype(activations.size()) idx;
					decltype(idx) idx_m1;

					for ( idx = 1; idx < (layers.size() - 1); ++idx ){

						idx_m1 = idx - 1;
						tmp = layers[idx_m1]*layers[idx];
						pre_activations.segment(offset,layers[idx]) = Eigen::Map<const MLMatrix<WeightType>>( ref_weights.data() + offset_weights, layers[idx], layers[idx_m1] )*activations.segment(offset-layers[idx_m1],layers[idx_m1]);
						offset_weights += tmp;
						pre_activations.segment(offset,layers[idx]) += ref_weights.segment(offset_weights,layers[idx]);	
						UnitProcessor<DROPOUT>::template process<HiddenLayerActivation,WeightType,RNG_TYPE>(pre_activations.segment(offset,layers[idx]),activations.segment(offset,layers[idx]),dropped.segment(offset,layers[idx]),rng,b_dist,inv_p);
						offset_weights += layers[idx];
						offset += layers[idx];  
					}
					
					idx_m1 = idx - 1;
					tmp = layers[idx_m1]*layers[idx];
					pre_activations.segment(offset,layers[idx]) = Eigen::Map<const MLMatrix<WeightType>>( ref_weights.data() + offset_weights, layers[idx], layers[idx_m1] )*activations.segment(offset-layers[idx_m1],layers[idx_m1]);
					offset_weights += tmp;
					pre_activations.segment(offset,layers[idx]) += weights.segment(offset_weights,layers[idx]);	
					activations.segment(offset,layers[idx]) = pre_activations.segment(offset,layers[idx]).unaryExpr(std::pointer_to_unary_function<WeightType,WeightType>(ActivationFunction<OutputLayerActivation>::evaluate));
					
				}
				// BACKPROPAGATION	
				/*!
				*	\brief 		Backpropagation algorithm
				*	\details	It assumes that a forward pass has already been performed.
				*/
				template< 	bool DROPOUT = false, 
							typename DERIVED,
							typename DERIVED_2,
							typename DERIVED_3 >
				void backpropagate( const Eigen::MatrixBase<DERIVED>& weights, const Eigen::MatrixBase<DERIVED_2>& gradient_output, Eigen::MatrixBase<DERIVED_3>& gradient_weights ) const{
					static_assert( std::is_same<typename DERIVED::Scalar,typename DERIVED_2::Scalar >::value && std::is_same<typename DERIVED::Scalar,typename DERIVED_3::Scalar >::value, "The vectors given as input has to have same scalar type!" );
					static_assert( std::is_same<typename DERIVED::Scalar,WeightType >::value, "The vectors' scalar type has to be consistent to the declared weight type!" );
					static_assert( (DERIVED::ColsAtCompileTime == 1) && (DERIVED_2::ColsAtCompileTime == 1) && (DERIVED_3::ColsAtCompileTime == 1), "The inputs have to be column vectors (or consistent structures)!");
					MLEARN_ASSERT( ( layers.head(layers.size()-1).dot(layers.tail(layers.size()-1)) + layers.tail(layers.size()-1).array().sum()) == weights.size(), "Number of weights is not consistent with number of units in the layers!" );
					MLEARN_ASSERT( gradient_output.size() == layers[layers.size()-1], "Output layer not consistent with the given gradient!" );
					MLEARN_ASSERT( gradient_weights.size() == weights.size(), "Weights size and respective gradient size not compatible!" );

					// 
					const DERIVED& ref_weights = weights.derived();
					
					decltype(activations.size()) idx = layers.size()-1;
					decltype(activations.size()) idx_m1 = idx-1;
					decltype(activations.size()) offset = layers.head(idx).array().sum();
					decltype(ref_weights.size()) offset_weights = ref_weights.size() - layers[idx];
					decltype(ref_weights.size()) tmp = layers[idx_m1]*layers[idx];

					// compute delta output layer and assign it to the gradientof the last biases
					InternalImpl::DerivativeWrapper<OutputLayerActivation>::template derive<WeightType>( pre_activations.segment(offset,layers[idx]),activations.segment(offset,layers[idx]),derivatives.head(layers[idx]) );
					gradient_weights.segment(offset_weights,layers[idx]) = gradient_output.cwiseProduct(derivatives.head(layers[idx]));
{'fraction_non_alphanumeric': 0.12199578355586789, 'fraction_numerical': 0.006558913094401499, 'mean_word_length': 4.590885280251441, 'pattern_counts': {'":': 0, '<': 135, '<?xml version=': 0, '>': 133, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5707244', 'n_tokens_mistral': 7935, 'n_tokens_neox': 6446, 'n_words': 1514}