<commit_before>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by <NAME> (<EMAIL>)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <boost/format.hpp>

#include "common.h"
#include "thread.h"

#include "generic_type.h"
#include "NUMA_vector.h"
#include "data_frame.h"
#include "mem_worker_thread.h"
#include "local_vec_store.h"
#include "matrix_store.h"

namespace fm
{

namespace detail
{

template<class T>
T ceil_divide(T v1, T v2)
{
	return ceil(((double) v1) / v2);
}

NUMA_vec_store::ptr NUMA_vec_store::cast(vec_store::ptr vec)
{
	if (!vec->is_in_mem()
			|| std::static_pointer_cast<mem_vec_store>(vec)->get_num_nodes() < 0) {
		BOOST_LOG_TRIVIAL(error) << "The vector isn't a NUMA vector";
		return NUMA_vec_store::ptr();
	}
	return std::static_pointer_cast<NUMA_vec_store>(vec);
}

NUMA_vec_store::NUMA_vec_store(size_t length, size_t num_nodes,
		const scalar_type &type): mem_vec_store(length, type), mapper(num_nodes)
{
	data.resize(num_nodes);
	size_t num_eles_per_node = ceil_divide(length, num_nodes);
	num_eles_per_node = ROUNDUP(num_eles_per_node, mapper.get_range_size());
	size_t size_per_node = num_eles_per_node * type.get_size();
	for (size_t node_id = 0; node_id < num_nodes; node_id++)
		data[node_id] = detail::raw_data_array(size_per_node, node_id);
}

void NUMA_vec_store::reset_data()
{
	reset_arrays(data);
}

namespace
{

class set_ele_operate: public detail::set_range_operate
{
	const set_vec_operate &op;
	const detail::NUMA_mapper &mapper;
	size_t entry_size;
public:
	set_ele_operate(const set_vec_operate &_op, const detail::NUMA_mapper &_mapper,
			size_t entry_size): op(_op), mapper(_mapper) {
		this->entry_size = entry_size;
	}

	void set(char *buf, size_t size, off_t local_off, int node_id) const {
		assert(local_off % entry_size == 0);
		assert(size % entry_size == 0);
		size_t num_eles = size / entry_size;
		size_t local_ele_idx = local_off / entry_size;
		size_t global_ele_idx = mapper.map2logical(node_id, local_ele_idx);
		op.set(buf, num_eles, global_ele_idx);
	}
};

}

void NUMA_vec_store::set_data(const set_vec_operate &op)
{
	set_ele_operate set_ele(op, mapper, get_entry_size());
	set_array_ranges(mapper, get_length(), get_entry_size(), set_ele, data);
}

const char *NUMA_vec_store::get_sub_arr(off_t start, off_t end) const
{
	// The start and end needs to fall into the same range.
	if (mapper.get_logical_range_id(start)
			!= mapper.get_logical_range_id(end - 1)) {
		BOOST_LOG_TRIVIAL(error) << boost::format(
				"[%1%, %2%) isn't in the same range") % start % end;
		return NULL;
	}

	std::pair<int, size_t> loc = mapper.map2physical(start);
	size_t off = loc.second * get_entry_size();
	return data[loc.first].get_raw() + off;
}

char *NUMA_vec_store::get_sub_arr(off_t start, off_t end)
{
	const NUMA_vec_store *const_this = this;
	return (char *) const_this->get_sub_arr(start, end);
}

bool NUMA_vec_store::is_sub_vec() const
{
	return false;
}

bool NUMA_vec_store::append(std::vector<vec_store::const_ptr>::const_iterator vec_it,
		std::vector<vec_store::const_ptr>::const_iterator vec_end)
{
	// TODO
	assert(0);
}

bool NUMA_vec_store::append(const vec_store &vec)
{
	// TODO
	assert(0);
}<commit_msg>[Matrix]: check boundary in accessing a NUMA vector.<commit_after>/*
 * Copyright 2014 Open Connectome Project (http://openconnecto.me)
 * Written by <NAME> (<EMAIL>)
 *
 * This file is part of FlashMatrix.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <boost/format.hpp>

#include "common.h"
#include "thread.h"

#include "generic_type.h"
#include "NUMA_vector.h"
#include "data_frame.h"
#include "mem_worker_thread.h"
#include "local_vec_store.h"
#include "matrix_store.h"

namespace fm
{

namespace detail
{

template<class T>
T ceil_divide(T v1, T v2)
{
	return ceil(((double) v1) / v2);
}

NUMA_vec_store::ptr NUMA_vec_store::cast(vec_store::ptr vec)
{
	if (!vec->is_in_mem()
			|| std::static_pointer_cast<mem_vec_store>(vec)->get_num_nodes() < 0) {
		BOOST_LOG_TRIVIAL(error) << "The vector isn't a NUMA vector";
		return NUMA_vec_store::ptr();
	}
	return std::static_pointer_cast<NUMA_vec_store>(vec);
}

NUMA_vec_store::NUMA_vec_store(size_t length, size_t num_nodes,
		const scalar_type &type): mem_vec_store(length, type), mapper(num_nodes)
{
	data.resize(num_nodes);
	size_t num_eles_per_node = ceil_divide(length, num_nodes);
	num_eles_per_node = ROUNDUP(num_eles_per_node, mapper.get_range_size());
	size_t size_per_node = num_eles_per_node * type.get_size();
	for (size_t node_id = 0; node_id < num_nodes; node_id++)
		data[node_id] = detail::raw_data_array(size_per_node, node_id);
}

void NUMA_vec_store::reset_data()
{
	reset_arrays(data);
}

namespace
{

class set_ele_operate: public detail::set_range_operate
{
	const set_vec_operate &op;
	const detail::NUMA_mapper &mapper;
	size_t entry_size;
public:
	set_ele_operate(const set_vec_operate &_op, const detail::NUMA_mapper &_mapper,
			size_t entry_size): op(_op), mapper(_mapper) {
		this->entry_size = entry_size;
	}

	void set(char *buf, size_t size, off_t local_off, int node_id) const {
		assert(local_off % entry_size == 0);
		assert(size % entry_size == 0);
		size_t num_eles = size / entry_size;
		size_t local_ele_idx = local_off / entry_size;
		size_t global_ele_idx = mapper.map2logical(node_id, local_ele_idx);
		op.set(buf, num_eles, global_ele_idx);
	}
};

}

void NUMA_vec_store::set_data(const set_vec_operate &op)
{
	set_ele_operate set_ele(op, mapper, get_entry_size());
	set_array_ranges(mapper, get_length(), get_entry_size(), set_ele, data);
}

const char *NUMA_vec_store::get_sub_arr(off_t start, off_t end) const
{
	// The start and end needs to fall into the same range.
	if (mapper.get_logical_range_id(start)
			!= mapper.get_logical_range_id(end - 1)) {
		BOOST_LOG_TRIVIAL(error) << boost::format(
				"[%1%, %2%) isn't in the same range") % start % end;
		return NULL;
	}
	if ((size_t) start >= get_length() || (size_t) end > get_length()) {
		BOOST_LOG_TRIVIAL(error)
			<< boost::format("[%1%, %2%) is out of boundary") % start % end;
		return NULL;
	}

	std::pair<int, size_t> loc = mapper.map2physical(start);
	size_t off = loc.second * get_entry_size();
	return data[loc.first].get_raw() + off;
}

char *NUMA_vec_store::get_sub_arr(off_t start, off_t end)
{
	const NUMA_vec_store *const_this = this;
	return (char *) const_this->get_sub_arr(start, end);
}

bool NUMA_vec_store::is_sub_vec() const
{
	return false;
}

bool NUMA_vec_store::append(std::vector<vec_store::const_ptr>::const_iterator vec_it,
		std::vector<vec_store::const_ptr>::const_iterator vec_end)
{
	// TODO
	assert(0);
}

bool NUMA_vec_store::append(const vec_store &vec)
{
	// TODO
	assert(0);
}
{'fraction_non_alphanumeric': 0.12561132561132562, 'fraction_numerical': 0.006177606177606178, 'mean_word_length': 5.4436152570480925, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4189797', 'n_tokens_mistral': 3036, 'n_tokens_neox': 2932, 'n_words': 681}