<commit_before>#ifndef NEURALNET_HPP
#define NEURALNET_HPP

#include <fstream>
#include <string>
#include <vector>
#include <algorithm>
#include <functional>
#include <random>

#include "matrix.hpp"

class Neuralnet
{
private:
	typedef Matrix<double> Mat;
	typedef std::vector<double> Vec;

	int BATCH_SIZE;
	double EPS, LAMBDA, MU, ALPHA, BETA, RHO, K, P;

	int num_layer;
	std::vector<int> num_unit;
	std::vector<Mat> W;
	
	std::vector<std::function<double(double)>> activate_func;
	std::vector<std::function<double(double)>> activate_diff_func;

	std::mt19937 m;
	std::uniform_real_distribution<double> d_rand;

	std::vector<Mat> calc_gradient (const std::vector<Mat>& u, const Mat& d, std::vector<Vec>& rho);
public:
	Neuralnet( const std::vector<int>& num_unit );
	void set_function ( const int& idx,
						const std::function<double(double)>& func,
						const std::function<double(double)>& diff_func );
	void set_EPS ( const double& EPS );
	void set_LAMBDA ( const double& LAMBDA );
	void set_MU ( const double& MU );
	void set_ALPHA ( const double& ALPHA );
	void set_BETA ( const double& BETA );
	void set_RHO ( const double& RHO );
	void set_K ( const double& K );
	void set_P ( const double& P );
	void set_BATCHSIZE ( const int& BATCH_SIZE );

	void set_W ( const std::string& filename );
	void set_W ( int idx, const Mat& w );
	Mat get_W ( int idx );
	
	void learning ( const std::vector<Vec>& x, const std::vector<Vec>& y,
					const int MAX_ITER = 1000 );

	Mat apply ( const Mat& X );
	Vec apply ( const Vec& x );

	void output_W ( const std::string& filename );
};

//////////////////// PRIVATE FUNCTION ////////////////////
std::vector<Neuralnet::Mat> Neuralnet::calc_gradient (
	const std::vector<Mat>& u, const Mat& d, std::vector<Vec>& rho )
{
	Mat delta(d.m, d.n);
	for( int i = 0; i < delta.m; ++i ){
		for( int j = 0; j < delta.n; ++j ){
			delta[i][j] = (activate_func[num_layer-1](u[num_layer-1][i+1][j]) - d[i][j]) *
				activate_diff_func[num_layer-1](u[num_layer-1][i+1][j]);
		}
	}
	
	std::vector<Mat> nabla_w(W.size());
	for( int i = 0; i < W.size(); ++i ) nabla_w[i] = Mat::zeros(W[i].m, W[i].n);
	for( int i = num_layer-2; i >= 0; --i ){
		for( int j = 0; j < nabla_w[i].m; ++j ){
			for( int k = 0; k < nabla_w[i].n; ++k ){
				for( int l = 0; l < delta.n; ++l ){
					nabla_w[i][j][k] += delta[j][l]*(
						i == 0 || k == 0 ? u[i][k][l] : activate_func[i-1](u[i][k][l])
						);
				}
			}
		}
		
		if( i == 0 ) continue;
		auto W_T = Mat::transpose(W[i]);
		auto nx_delta = W_T*delta;
		delta = Mat(nx_delta.m-1, nx_delta.n);

		if( BETA > 1.0E-10 ){
			const double R_LAMBDA = 0.9;
			for( int j = 0; j < u[i].m-1; ++j ){
				double tmp_rho = 0.0;
				for( int k = 0; k < u[i].n; ++k ){
					tmp_rho += activate_func[i-1](u[i][1+j][k]);
				}
				tmp_rho /= u[i].n;
				rho[i][j] = R_LAMBDA*rho[i][j] + (1-R_LAMBDA)*tmp_rho;
			}
		}
		for( int j = 0; j < delta.m; ++j ){
			double KL = (BETA > 1.0E-10 ? (1-RHO)/(1-rho[i][j]) - RHO/rho[i][j] : 0.0);
			for( int k = 0; k < delta.n; ++k ){
				delta[j][k] = (nx_delta[j+1][k] + BETA*KL)*activate_diff_func[i-1](u[i][j+1][k]);
			}
		}
	}

	return nabla_w;
}

//////////////////// PUBLIC FUNCTION ////////////////////
Neuralnet::Neuralnet( const std::vector<int>& num_unit )
	:num_layer(num_unit.size()), num_unit(num_unit),
	 EPS(1.0E-1), LAMBDA(1.0E-5), MU(0.0), ALPHA(1.0), BETA(0.0), RHO(-1.0), BATCH_SIZE(1), K(1.0)
{
	m = std::mt19937(time(NULL));
	for( int i = 1; i < this->num_layer; ++i ){
		double tmp = sqrt(6.0/(num_unit[i] + num_unit[i-1]));
		d_rand = std::uniform_real_distribution<double>(-tmp, tmp);
		W.push_back( Mat(num_unit[i], num_unit[i-1]+1) );

		auto& w = *W.rbegin();
		for( int j = 0; j < w.m; ++j )
			for( int k = 1; k < w.n; ++k )
				w[j][k] = d_rand(m);
	}

	activate_func.resize( this->num_layer );
	activate_diff_func.resize( this->num_layer );
}

void Neuralnet::set_function ( const int& idx,
								  const std::function<double(double)>& func,
								  const std::function<double(double)>& diff_func )
{
	activate_func[idx] = func;
	activate_diff_func[idx] = diff_func;
}

void Neuralnet::set_EPS ( const double& EPS )
{
	this->EPS = EPS;
}

void Neuralnet::set_LAMBDA ( const double& LAMBDA )
{
	this->LAMBDA = LAMBDA;
}

void Neuralnet::set_MU ( const double& MU )
{
	this->MU = MU;
}
	
void Neuralnet::set_ALPHA ( const double& ALPHA )
{
	this->ALPHA = ALPHA;
}

void Neuralnet::set_BETA ( const double& BETA )
{
	this->BETA = BETA;
}

void Neuralnet::set_RHO ( const double& RHO )
{
	this->RHO = RHO;
}

void Neuralnet::set_BATCHSIZE ( const int& BATCH_SIZE )
{
	this->BATCH_SIZE = BATCH_SIZE;
}

void Neuralnet::set_K ( const double& K )
{
	this->K = K;
}

void Neuralnet::set_P ( const double& P )
{
	this->P = P;
}

void Neuralnet::set_W ( const std::string& filename )
{<commit_msg>Added variables and functions related to drop out.<commit_after>#ifndef NEURALNET_HPP
#define NEURALNET_HPP

#include <fstream>
#include <string>
#include <vector>
#include <algorithm>
#include <functional>
#include <random>

#include "matrix.hpp"

class Neuralnet
{
private:
	typedef Matrix<double> Mat;
	typedef std::vector<double> Vec;

	int BATCH_SIZE;
	double EPS, LAMBDA, MU, BETA, RHO, DO_PROB;

	int num_layer;
	std::vector<int> num_unit;
	std::vector<Mat> W;
	std::vector<Vec> mask;
	
	std::vector<std::function<double(double)>> activate_func;
	std::vector<std::function<double(double)>> activate_diff_func;

	std::mt19937 m;
	std::uniform_real_distribution<double> d_rand;

	std::vector<Mat> calc_gradient (const std::vector<Mat>& u, const Mat& d, std::vector<Vec>& rho);
public:
	Neuralnet( const std::vector<int>& num_unit );
	void set_function ( const int& idx,
						const std::function<double(double)>& func,
						const std::function<double(double)>& diff_func );
	void set_EPS ( const double& EPS );
	void set_LAMBDA ( const double& LAMBDA );
	void set_MU ( const double& MU );
	void set_BETA ( const double& BETA );
	void set_RHO ( const double& RHO );
	void set_PROB ( const double& PROB );
	void set_BATCHSIZE ( const int& BATCH_SIZE );

	void set_W ( const std::string& filename );
	void set_W ( int idx, const Mat& w );
	Mat get_W ( int idx );
	
	void learning ( const std::vector<Vec>& x, const std::vector<Vec>& y,
					const int MAX_ITER = 1000 );

	Mat apply ( const Mat& X );
	Vec apply ( const Vec& x );

	void output_W ( const std::string& filename );
};

//////////////////// PRIVATE FUNCTION ////////////////////
std::vector<Neuralnet::Mat> Neuralnet::calc_gradient (
	const std::vector<Mat>& u, const Mat& d, std::vector<Vec>& rho )
{
	Mat delta(d.m, d.n);
	for( int i = 0; i < delta.m; ++i ){
		for( int j = 0; j < delta.n; ++j ){
			delta[i][j] = (activate_func[num_layer-1](u[num_layer-1][i+1][j]) - d[i][j]) *
				activate_diff_func[num_layer-1](u[num_layer-1][i+1][j]);
		}
	}
	
	std::vector<Mat> nabla_w(W.size());
	for( int i = 0; i < W.size(); ++i ) nabla_w[i] = Mat::zeros(W[i].m, W[i].n);
	for( int i = num_layer-2; i >= 0; --i ){
		for( int j = 0; j < nabla_w[i].m; ++j ){
			for( int k = 0; k < nabla_w[i].n; ++k ){
				for( int l = 0; l < delta.n; ++l ){
					nabla_w[i][j][k] += delta[j][l]*(
						i == 0 || k == 0 ? u[i][k][l] : activate_func[i-1](u[i][k][l])
						);
				}
			}
		}
		
		if( i == 0 ) continue;
		auto W_T = Mat::transpose(W[i]);
		auto nx_delta = W_T*delta;
		delta = Mat(nx_delta.m-1, nx_delta.n);

		if( BETA > 1.0E-10 ){
			const double R_LAMBDA = 0.9;
			for( int j = 0; j < u[i].m-1; ++j ){
				double tmp_rho = 0.0;
				for( int k = 0; k < u[i].n; ++k ){
					tmp_rho += activate_func[i-1](u[i][1+j][k]);
				}
				tmp_rho /= u[i].n;
				rho[i][j] = R_LAMBDA*rho[i][j] + (1-R_LAMBDA)*tmp_rho;
			}
		}
		for( int j = 0; j < delta.m; ++j ){
			double KL = (BETA > 1.0E-10 ? (1-RHO)/(1-rho[i][j]) - RHO/rho[i][j] : 0.0);
			for( int k = 0; k < delta.n; ++k ){
				delta[j][k] = (nx_delta[j+1][k] + BETA*KL)*activate_diff_func[i-1](u[i][j+1][k]);
			}
		}
	}

	return nabla_w;
}

//////////////////// PUBLIC FUNCTION ////////////////////
Neuralnet::Neuralnet( const std::vector<int>& num_unit )
	:num_layer(num_unit.size()), num_unit(num_unit),
	 EPS(1.0E-1), LAMBDA(1.0E-5), BETA(0.0), RHO(-1.0), BATCH_SIZE(1)
{
	m = std::mt19937(time(NULL));
	for( int i = 1; i < this->num_layer; ++i ){
		double tmp = sqrt(6.0/(num_unit[i] + num_unit[i-1]));
		d_rand = std::uniform_real_distribution<double>(-tmp, tmp);
		W.push_back( Mat(num_unit[i], num_unit[i-1]+1) );

		auto& w = *W.rbegin();
		for( int j = 0; j < w.m; ++j )
			for( int k = 1; k < w.n; ++k )
				w[j][k] = d_rand(m);
	}

	activate_func.resize( this->num_layer );
	activate_diff_func.resize( this->num_layer );
}

void Neuralnet::set_function ( const int& idx,
								  const std::function<double(double)>& func,
								  const std::function<double(double)>& diff_func )
{
	activate_func[idx] = func;
	activate_diff_func[idx] = diff_func;
}

void Neuralnet::set_EPS ( const double& EPS )
{
	this->EPS = EPS;
}

void Neuralnet::set_LAMBDA ( const double& LAMBDA )
{
	this->LAMBDA = LAMBDA;
}

void Neuralnet::set_MU ( const double& MU )
{
	this->MU = MU;
}

void Neuralnet::set_BETA ( const double& BETA )
{
	this->BETA = BETA;
}

void Neuralnet::set_RHO ( const double& RHO )
{
	this->RHO = RHO;
}

void Neuralnet::set_BATCHSIZE ( const int& BATCH_SIZE )
{
	this->BATCH_SIZE = BATCH_SIZE;
}

void Neuralnet::set_PROB ( const double& PROB )
{
	this->DO_PROB = PROB;
}

void Neuralnet::set_W ( const std::string& filename )
{
{'fraction_non_alphanumeric': 0.20502533783783783, 'fraction_numerical': 0.01731418918918919, 'mean_word_length': 3.5088053307948597, 'pattern_counts': {'":': 0, '<': 92, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '291116', 'n_tokens_mistral': 4413, 'n_tokens_neox': 3950, 'n_words': 896}