<commit_before>#include <signal.h>
#include <gperftools/profiler.h>

#include <boost/format.hpp>

#include "matrix_config.h"
#include "sparse_matrix.h"
#include "dense_matrix.h"

using namespace fm;

size_t length = 4096L * 1024 * 1024;

template<class T>
class mat_init: public type_set_operate<T>
{
public:
	virtual void set(T *arr, size_t num_eles, off_t row_idx,
			            off_t col_idx) const {
		for (size_t i = 0; i < num_eles; i++)
			arr[i] = i;
	}
};

void test_mapply()
{
	struct timeval start, end;
	typedef size_t ele_type;
	size_t height = 1024L * 1024 * 1024;
	size_t width = 8;

	dense_matrix::ptr mat1 = dense_matrix::create(height, width,
			matrix_layout_t::L_ROW, get_scalar_type<ele_type>(),
			mat_init<ele_type>(), matrix_conf.get_num_nodes());
	dense_matrix::ptr mat2 = dense_matrix::create(height, width,
			matrix_layout_t::L_ROW, get_scalar_type<ele_type>(),
			mat_init<ele_type>(), matrix_conf.get_num_nodes());
	bulk_operate::const_ptr min
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::MIN));
	bulk_operate::const_ptr max
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::MAX));
	bulk_operate::const_ptr add
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::ADD));
	bulk_operate::const_ptr sub
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::SUB));

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, min);
		res1->materialize_self();
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, max);
		res2->materialize_self();
		gettimeofday(&end, NULL);
		printf("separate min+max takes %.3f seconds\n", time_diff(start, end));
	}

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, min);
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, max);
		std::vector<dense_matrix::ptr> ress(2);
		ress[0] = res1;
		ress[1] = res2;
		materialize(ress);
		gettimeofday(&end, NULL);
		printf("min+max takes %.3f seconds\n", time_diff(start, end));
	}

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, add);
		res1->materialize_self();
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, sub);
		res2->materialize_self();
		gettimeofday(&end, NULL);
		printf("separate add+sub takes %.3f seconds\n", time_diff(start, end));
	}

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, add);
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, sub);
		std::vector<dense_matrix::ptr> ress(2);
		ress[0] = res1;
		ress[1] = res2;
		materialize(ress);
		gettimeofday(&end, NULL);
		printf("add+sub takes %.3f seconds\n", time_diff(start, end));
	}
}

void test_conv_layout()
{
	struct timeval start, end;
	typedef size_t ele_type;
	size_t height = 1024 * 1024 * 1024;
	size_t width = 8;

	dense_matrix::ptr mat = dense_matrix::create(height, width,
			matrix_layout_t::L_ROW, get_scalar_type<ele_type>(),
			mat_init<ele_type>(), matrix_conf.get_num_nodes());
	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res = mat->conv2(matrix_layout_t::L_COL);
		res->materialize_self();
		gettimeofday(&end, NULL);
		printf("conv layout takes %.3f seconds\n", time_diff(start, end));
	}
}

void test_copy()
{
	struct timeval start, end;<commit_msg>[Matrix]: measure the perf of conv layout.<commit_after>#include <signal.h>
#include <gperftools/profiler.h>

#include <boost/format.hpp>

#include "matrix_config.h"
#include "sparse_matrix.h"
#include "dense_matrix.h"

using namespace fm;

size_t length = 4096L * 1024 * 1024;

template<class T>
class mat_init: public type_set_operate<T>
{
public:
	virtual void set(T *arr, size_t num_eles, off_t row_idx,
			            off_t col_idx) const {
		for (size_t i = 0; i < num_eles; i++)
			arr[i] = i;
	}
};

void test_mapply()
{
	struct timeval start, end;
	typedef size_t ele_type;
	size_t height = 1024L * 1024 * 1024;
	size_t width = 8;

	dense_matrix::ptr mat1 = dense_matrix::create(height, width,
			matrix_layout_t::L_ROW, get_scalar_type<ele_type>(),
			mat_init<ele_type>(), matrix_conf.get_num_nodes());
	dense_matrix::ptr mat2 = dense_matrix::create(height, width,
			matrix_layout_t::L_ROW, get_scalar_type<ele_type>(),
			mat_init<ele_type>(), matrix_conf.get_num_nodes());
	bulk_operate::const_ptr min
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::MIN));
	bulk_operate::const_ptr max
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::MAX));
	bulk_operate::const_ptr add
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::ADD));
	bulk_operate::const_ptr sub
		= bulk_operate::conv2ptr(*mat1->get_type().get_basic_ops().get_op(basic_ops::op_idx::SUB));

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, min);
		res1->materialize_self();
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, max);
		res2->materialize_self();
		gettimeofday(&end, NULL);
		printf("separate min+max takes %.3f seconds\n", time_diff(start, end));
	}

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, min);
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, max);
		std::vector<dense_matrix::ptr> ress(2);
		ress[0] = res1;
		ress[1] = res2;
		materialize(ress);
		gettimeofday(&end, NULL);
		printf("min+max takes %.3f seconds\n", time_diff(start, end));
	}

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, add);
		res1->materialize_self();
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, sub);
		res2->materialize_self();
		gettimeofday(&end, NULL);
		printf("separate add+sub takes %.3f seconds\n", time_diff(start, end));
	}

	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res1 = mat1->mapply2(*mat2, add);
		dense_matrix::ptr res2 = mat1->mapply2(*mat2, sub);
		std::vector<dense_matrix::ptr> ress(2);
		ress[0] = res1;
		ress[1] = res2;
		materialize(ress);
		gettimeofday(&end, NULL);
		printf("add+sub takes %.3f seconds\n", time_diff(start, end));
	}
}

void test_conv_layout()
{
	struct timeval start, end;
	typedef size_t ele_type;
	size_t height = 1024 * 1024 * 256;
	size_t width = 32;

	dense_matrix::ptr mat = dense_matrix::create(height, width,
			matrix_layout_t::L_ROW, get_scalar_type<ele_type>(),
			mat_init<ele_type>(), matrix_conf.get_num_nodes());
	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res = mat->conv2(matrix_layout_t::L_COL);
		res->materialize_self();
		gettimeofday(&end, NULL);
		printf("conv layout (row to col) takes %.3f seconds\n",
				time_diff(start, end));
	}

	mat = dense_matrix::create(height, width, matrix_layout_t::L_COL,
			get_scalar_type<ele_type>(), mat_init<ele_type>(),
			matrix_conf.get_num_nodes());
	for (size_t i = 0; i < 5; i++) {
		gettimeofday(&start, NULL);
		dense_matrix::ptr res = mat->conv2(matrix_layout_t::L_ROW);
		res->materialize_self();
		gettimeofday(&end, NULL);
		printf("conv layout (col to row) takes %.3f seconds\n",
				time_diff(start, end));
	}
}

void test_copy()
{
	struct timeval start, end;
{'fraction_non_alphanumeric': 0.16808424463345484, 'fraction_numerical': 0.030511678142297827, 'mean_word_length': 5.419410745233969, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4533333', 'n_tokens_mistral': 3222, 'n_tokens_neox': 2956, 'n_words': 570}