<commit_before>#ifndef NEURALNET_HPP
#define NEURALNET_HPP

#include <fstream>
#include <string>
#include <vector>
#include <algorithm>
#include <functional>
#include <memory>
#include <random>

#include "Layer.hpp"
#include "Convolutional.hpp"
#include "Function.hpp"
#include "matrix.hpp"

#ifdef USE_MPI
#include <mpi.h>
#endif

class Neuralnet
{
private:
	typedef Matrix<double> Mat;
	typedef std::vector<double> Vec;

	const double adam_beta = 0.9, adam_gamma = 0.999, adam_eps = 1.0E-8;
	std::vector<std::vector<std::vector<Mat>>> adam_v, adam_r;
	double adam_beta_ = 1.0, adam_gamma_ = 1.0;

	int BATCH_SIZE, UPDATE_ITER;
	double EPS, LAMBDA;

	std::shared_ptr<LossFunction> loss;
	std::vector<std::shared_ptr<Layer>> layer;
	
	std::mt19937 mt;
	std::uniform_real_distribution<double> d_rand;

#ifdef USE_MPI
	MPI_Comm outer_world, inner_world;
#endif

	std::vector<std::vector<std::vector<Mat>>> calc_gradient (const std::vector<std::vector<Mat>>& U, const std::vector<Mat>& d);
	void check_gradient ( int cnt, const std::vector<int>& idx, const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y, const std::vector<std::vector<std::vector<Mat>>>& nabla_w );
public:
	Neuralnet( const std::shared_ptr<LossFunction>& loss );
#ifdef USE_MPI
	Neuralnet( const std::shared_ptr<LossFunction>& loss, MPI_Comm outer_world, MPI_Comm inner_world );
#endif

	void set_EPS ( const double& EPS );
	void set_LAMBDA ( const double& LAMBDA );
	void set_BATCHSIZE ( const int& BATCH_SIZE );
	void set_UPDATEITER ( const int& UPDATE_ITER );

	void add_layer( const std::shared_ptr<Layer>& layer );

#ifdef USE_MPI
	void averaging ();
#endif
	void learning ( const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y,
					const int MAX_ITER = 1000,
					const std::function<void(Neuralnet&, const int, const std::vector<Mat>&, const std::vector<Mat>&)>& each_func
					= [](Neuralnet& nn, const int epoch, const std::vector<Mat>& x, const std::vector<Mat>& d) -> void {} );

	std::vector<Mat> apply ( const std::vector<Mat>& X ) const;
	std::vector<std::vector<Vec>> apply ( const std::vector<std::vector<Vec>>& x ) const;

	void print_cost ( const std::vector<Mat>& x, const std::vector<Mat>& y ) const;
	void print_cost ( const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y ) const;
	void print_weight () const;
	void print_gradient () const;

	void set_W ( const std::string& filename );
	void output_W ( const std::string& filename ) const;
};

//////////////////// PRIVATE FUNCTION ////////////////////
std::vector<std::vector<std::vector<Neuralnet::Mat>>> Neuralnet::calc_gradient (
	const std::vector<std::vector<Mat>>& U, const std::vector<Mat>& d )
{
	const int num_layer = layer.size();
	
	std::vector<Mat> delta(d.size());
	for( int i = 0; i < d.size(); ++i ) delta[i] = Mat(d[i].m, d[i].n);

	std::shared_ptr<Function> f = layer[num_layer-1]->get_function();
	for( int i = 0; i < d.size(); ++i )
		delta[i] = Mat::hadamard((*loss)((*f)(U[num_layer][i], false), d[i], true),
								 (*f)(U[num_layer][i], true));

	std::vector<std::vector<std::vector<Mat>>> nabla_w(num_layer);
	for( int i = num_layer-1; i >= 0; --i ){
		nabla_w[i] = layer[i]->calc_gradient(U[i], delta);
		
		if( i == 0 ) continue;
		delta = layer[i]->calc_delta(U[i], delta);
	}
	return nabla_w;
}

void Neuralnet::check_gradient ( int cnt, const std::vector<int>& idx, const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y, const std::vector<std::vector<std::vector<Mat>>>& nabla_w )
{
	int rank = 0;
	int num_layer = this->layer.size();

#ifdef USE_MPI
	MPI_Comm_rank(inner_world, &rank);
#endif
	
	// Calculate gradient numerically for confirmation of computing
	for( int i = 0; i < num_layer; ++i ){
		if( rank == 0 ) printf("\tlayer %d\n", i);
		auto W = layer[i]->get_W();
		std::vector<std::vector<Vec>> X(BATCH_SIZE);
		for( int j = 0; j < BATCH_SIZE; ++j ) X[j] = x[idx[cnt+j]];
		for( int j = 0; j < std::min(2, (int)W.size()); ++j ){ // num_map
			for( int k = 0; k < std::min(2, (int)W[j].size()); ++k ){ // prev_num_map
				for( int l = 0; l < std::min(2, (int)W[j][k].m); ++l ){
					for( int m = 0; m < std::min(2, (int)W[j][k].n); ++m ){
						auto tmp = 1.0E-8;//*(std::abs(W[j][k](l,m)) < 1.0E-3 ? 1.0 : std::abs(W[j][k](l,m)));;

						if( layer[i]->get_num_map() != 1 || rank == 0 ){
							if( layer[i]->get_num_map() != 1 && l == 0 && m == 0 ){ // for check bias
								Convolutional* conv = (Convolutional*)layer[i].get();
								conv->bias[j] += tmp;
							}
							else{
								W[j][k](l,m) += tmp;
								layer[i]->set_W(W);
							}
						}
						double E1 = 0.0;
						auto tmp1 = apply(X);
						for( int n = 0; n < tmp1[0].size(); ++n )
							for( int o = 0; o < BATCH_SIZE; ++o ){
								E1 += (*loss)(Mat(tmp1[o][n]), Mat(y[idx[cnt+o]][n]), false)(0,0);
							}

						if( layer[i]->get_num_map() != 1 || rank == 0 ){
							if( layer[i]->get_num_map() != 1 && l == 0 && m == 0 ){
								Convolutional* conv = (Convolutional*)layer[i].get();
								conv->bias[j] -= tmp;
							}
							else{
								W[j][k](l,m) -= tmp;
								layer[i]->set_W(W);
							}
						}
						double E2 = 0.0;
						auto tmp2 = apply(X);
						for( int n = 0; n < tmp2[0].size(); ++n )
							for( int o = 0; o < BATCH_SIZE; ++o )
								E2 += (*loss)(Mat(tmp2[o][n]), Mat(y[idx[cnt+o]][n]), false)(0,0);

						if( rank == 0 ){
							double grad = nabla_w[i][j][k](l,m);
							if( layer[i]->get_num_map() != 1 && l == 0 && m == 0 ){
								Convolutional* conv = (Convolutional*)layer[i].get();
								grad = conv->d_bias[j];
							}								<commit_msg>Removed check gradient of bias in convolution.<commit_after>#ifndef NEURALNET_HPP
#define NEURALNET_HPP

#include <fstream>
#include <string>
#include <vector>
#include <algorithm>
#include <functional>
#include <memory>
#include <random>

#include "Layer.hpp"
#include "Convolutional.hpp"
#include "Function.hpp"
#include "matrix.hpp"

#ifdef USE_MPI
#include <mpi.h>
#endif

class Neuralnet
{
private:
	typedef Matrix<double> Mat;
	typedef std::vector<double> Vec;

	const double adam_beta = 0.9, adam_gamma = 0.999, adam_eps = 1.0E-8;
	std::vector<std::vector<std::vector<Mat>>> adam_v, adam_r;
	double adam_beta_ = 1.0, adam_gamma_ = 1.0;

	int BATCH_SIZE, UPDATE_ITER;
	double EPS, LAMBDA;

	std::shared_ptr<LossFunction> loss;
	std::vector<std::shared_ptr<Layer>> layer;
	
	std::mt19937 mt;
	std::uniform_real_distribution<double> d_rand;

#ifdef USE_MPI
	MPI_Comm outer_world, inner_world;
#endif

	std::vector<std::vector<std::vector<Mat>>> calc_gradient (const std::vector<std::vector<Mat>>& U, const std::vector<Mat>& d);
	void check_gradient ( int cnt, const std::vector<int>& idx, const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y, const std::vector<std::vector<std::vector<Mat>>>& nabla_w );
public:
	Neuralnet( const std::shared_ptr<LossFunction>& loss );
#ifdef USE_MPI
	Neuralnet( const std::shared_ptr<LossFunction>& loss, MPI_Comm outer_world, MPI_Comm inner_world );
#endif

	void set_EPS ( const double& EPS );
	void set_LAMBDA ( const double& LAMBDA );
	void set_BATCHSIZE ( const int& BATCH_SIZE );
	void set_UPDATEITER ( const int& UPDATE_ITER );

	void add_layer( const std::shared_ptr<Layer>& layer );

#ifdef USE_MPI
	void averaging ();
#endif
	void learning ( const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y,
					const int MAX_ITER = 1000,
					const std::function<void(Neuralnet&, const int, const std::vector<Mat>&, const std::vector<Mat>&)>& each_func
					= [](Neuralnet& nn, const int epoch, const std::vector<Mat>& x, const std::vector<Mat>& d) -> void {} );

	std::vector<Mat> apply ( const std::vector<Mat>& X ) const;
	std::vector<std::vector<Vec>> apply ( const std::vector<std::vector<Vec>>& x ) const;

	void print_cost ( const std::vector<Mat>& x, const std::vector<Mat>& y ) const;
	void print_cost ( const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y ) const;
	void print_weight () const;
	void print_gradient () const;

	void set_W ( const std::string& filename );
	void output_W ( const std::string& filename ) const;
};

//////////////////// PRIVATE FUNCTION ////////////////////
std::vector<std::vector<std::vector<Neuralnet::Mat>>> Neuralnet::calc_gradient (
	const std::vector<std::vector<Mat>>& U, const std::vector<Mat>& d )
{
	const int num_layer = layer.size();
	
	std::vector<Mat> delta(d.size());
	for( int i = 0; i < d.size(); ++i ) delta[i] = Mat(d[i].m, d[i].n);

	std::shared_ptr<Function> f = layer[num_layer-1]->get_function();
	for( int i = 0; i < d.size(); ++i )
		delta[i] = Mat::hadamard((*loss)((*f)(U[num_layer][i], false), d[i], true),
								 (*f)(U[num_layer][i], true));

	std::vector<std::vector<std::vector<Mat>>> nabla_w(num_layer);
	for( int i = num_layer-1; i >= 0; --i ){
		nabla_w[i] = layer[i]->calc_gradient(U[i], delta);
		
		if( i == 0 ) continue;
		delta = layer[i]->calc_delta(U[i], delta);
	}
	return nabla_w;
}

void Neuralnet::check_gradient ( int cnt, const std::vector<int>& idx, const std::vector<std::vector<Vec>>& x, const std::vector<std::vector<Vec>>& y, const std::vector<std::vector<std::vector<Mat>>>& nabla_w )
{
	int rank = 0;
	int num_layer = this->layer.size();

#ifdef USE_MPI
	MPI_Comm_rank(inner_world, &rank);
#endif
	
	// Calculate gradient numerically for confirmation of computing
	for( int i = 0; i < num_layer; ++i ){
		if( rank == 0 ) printf("\tlayer %d\n", i);
		auto W = layer[i]->get_W();
		std::vector<std::vector<Vec>> X(BATCH_SIZE);
		for( int j = 0; j < BATCH_SIZE; ++j ) X[j] = x[idx[cnt+j]];
		for( int j = 0; j < std::min(2, (int)W.size()); ++j ){ // num_map
			for( int k = 0; k < std::min(2, (int)W[j].size()); ++k ){ // prev_num_map
				for( int l = 0; l < std::min(2, (int)W[j][k].m); ++l ){
					for( int m = 0; m < std::min(2, (int)W[j][k].n); ++m ){
						auto tmp = 1.0E-6*(std::abs(W[j][k](l,m)) < 1.0E-3 ? 1.0 : std::abs(W[j][k](l,m)));;

						if( layer[i]->get_num_map() != 1 || rank == 0 ){
							W[j][k](l,m) += tmp;
							layer[i]->set_W(W);
						}
						double E1 = 0.0;
						auto tmp1 = apply(X);
						for( int n = 0; n < tmp1[0].size(); ++n )
							for( int o = 0; o < BATCH_SIZE; ++o ){
								E1 += (*loss)(Mat(tmp1[o][n]), Mat(y[idx[cnt+o]][n]), false)(0,0);
							}

						if( layer[i]->get_num_map() != 1 || rank == 0 ){
							W[j][k](l,m) -= tmp;
							layer[i]->set_W(W);
						}
						double E2 = 0.0;
						auto tmp2 = apply(X);
						for( int n = 0; n < tmp2[0].size(); ++n )
							for( int o = 0; o < BATCH_SIZE; ++o )
								E2 += (*loss)(Mat(tmp2[o][n]), Mat(y[idx[cnt+o]][n]), false)(0,0);

						if( rank == 0 ){
							double grad = nabla_w[i][j][k](l,m);

{'fraction_non_alphanumeric': 0.210296440851222, 'fraction_numerical': 0.01514729114394573, 'mean_word_length': 4.0055813953488375, 'pattern_counts': {'":': 0, '<': 181, '<?xml version=': 0, '>': 183, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6052737', 'n_tokens_mistral': 4754, 'n_tokens_neox': 4100, 'n_words': 969}