<commit_before>// Copyright (C) 2011 <NAME> <<EMAIL>>

// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
//
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.


#ifndef SIGPX_VECTOROPS_H
#define SIGPX_VECTOROPS_H

#include <iostream>
#include <vector>
#include <cassert>
#include <stdint.h>


namespace sigpx {


class VectorOps {
public:
	template<typename tp_Type> static std::ostream& printT(std::ostream &os, const std::vector<tp_Type> &v);

	template<typename tp_Type> static void stuffT(std::vector<tp_Type> &stuffed, std::vector<int32_t> &sizes, const std::vector<tp_Type> &v);

	template<typename tp_Type> static void unstuffT(const std::vector<tp_Type> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<tp_Type> &v);

	static std::ostream& print(std::ostream &os, const std::vector<int8_t> &v);
	static std::ostream& print(std::ostream &os, const std::vector<uint8_t> &v);
	static void stuff(std::vector<char> &stuffed, std::vector<int32_t> &sizes, const std::vector<char> &v);
	static void unstuff(const std::vector<char> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<char> &v);

	static std::ostream& print(std::ostream &os, const std::vector<int16_t> &v);
	static std::ostream& print(std::ostream &os, const std::vector<uint16_t> &v);
	static void stuff(std::vector<int16_t> &stuffed, std::vector<int32_t> &sizes, const std::vector<int16_t> &v);
	static void unstuff(const std::vector<int16_t> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<int16_t> &v);

	static std::ostream& print(std::ostream &os, const std::vector<int32_t> &v);
	static std::ostream& print(std::ostream &os, const std::vector<uint32_t> &v);
	static void stuff(std::vector<int32_t> &stuffed, std::vector<int32_t> &sizes, const std::vector<int32_t> &v);
	static void unstuff(const std::vector<int32_t> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<int32_t> &v);
};


template<typename tp_Type> std::ostream& VectorOps::printT(std::ostream &os, const std::vector<tp_Type> &v) {
	using namespace std;
	cout << "(";
	for (size_t i = 0; i < v.size(); ++i) cout << ((i > 0) ? ", " : "") << v[i];
	cout << ")";
	return cout;
}


template<typename tp_Type> void VectorOps::stuffT(std::vector<tp_Type> &stuffed, std::vector<int32_t> &sizes, const std::vector<tp_Type> &v) {
	size_t offset = stuffed.size();
	size_t requiredSize = offset + v.size();
	if (stuffed.capacity() < requiredSize)
		stuffed.reserve(std::max(stuffed.capacity() * 2, requiredSize));
	stuffed.resize(requiredSize);
	for (size_t i = 0; i < v.size(); ++i) stuffed[offset + i] = v[i];
	if (sizes.capacity() < sizes.size() + 1) sizes.reserve(std::max(sizes.capacity() * 2, sizes.size() + 1));
	sizes.push_back(v.size());
}

<commit_msg>Added VectorOps::copy methods<commit_after>// Copyright (C) 2011 <NAME> <<EMAIL>>

// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
//
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.


#ifndef SIGPX_VECTOROPS_H
#define SIGPX_VECTOROPS_H

#include <iostream>
#include <vector>
#include <cassert>
#include <stdint.h>


namespace sigpx {


class VectorOps {
public:
	template<typename tp_Type> static std::ostream& printT(std::ostream &os, const std::vector<tp_Type> &v);

	template<typename tp_Type> static void stuffT(std::vector<tp_Type> &stuffed, std::vector<int32_t> &sizes, const std::vector<tp_Type> &v);

	template<typename tp_Type> static void unstuffT(const std::vector<tp_Type> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<tp_Type> &v);

	template<typename tp_Type, typename tp_Type2> inline static void copyT(const std::vector<tp_Type> &src, std::vector<tp_Type2> &trg) {
		trg.resize(src.size());
		for (size_t i = 0; i < src.size(); ++i) trg[i] = src[i];
	}

	static std::ostream& print(std::ostream &os, const std::vector<int8_t> &v);
	static std::ostream& print(std::ostream &os, const std::vector<uint8_t> &v);
	static void stuff(std::vector<char> &stuffed, std::vector<int32_t> &sizes, const std::vector<char> &v);
	static void unstuff(const std::vector<char> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<char> &v);

	static std::ostream& print(std::ostream &os, const std::vector<int16_t> &v);
	static std::ostream& print(std::ostream &os, const std::vector<uint16_t> &v);
	static void stuff(std::vector<int16_t> &stuffed, std::vector<int32_t> &sizes, const std::vector<int16_t> &v);
	static void unstuff(const std::vector<int16_t> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<int16_t> &v);

	static std::ostream& print(std::ostream &os, const std::vector<int32_t> &v);
	static std::ostream& print(std::ostream &os, const std::vector<uint32_t> &v);
	static void stuff(std::vector<int32_t> &stuffed, std::vector<int32_t> &sizes, const std::vector<int32_t> &v);
	static void unstuff(const std::vector<int32_t> &stuffed, const std::vector<int32_t> &sizes, size_t i, std::vector<int32_t> &v);

	inline static void copy(const std::vector<int8_t> &src, std::vector<int8_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int8_t> &src, std::vector<uint8_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<uint8_t> &src, std::vector<int8_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int16_t> &src, std::vector<int16_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int16_t> &src, std::vector<uint16_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int16_t> &src, std::vector<int32_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int16_t> &src, std::vector<float> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int16_t> &src, std::vector<double> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<uint16_t> &src, std::vector<int16_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int32_t> &src, std::vector<int32_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int32_t> &src, std::vector<uint32_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int32_t> &src, std::vector<int16_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int32_t> &src, std::vector<float> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<int32_t> &src, std::vector<double> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<uint32_t> &src, std::vector<int32_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<float> &src, std::vector<float> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<float> &src, std::vector<double> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<float> &src, std::vector<int16_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<float> &src, std::vector<int32_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<double> &src, std::vector<double> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<double> &src, std::vector<float> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<double> &src, std::vector<int16_t> &trg) { copyT(src, trg); }
	inline static void copy(const std::vector<double> &src, std::vector<int32_t> &trg) { copyT(src, trg); }
};


template<typename tp_Type> std::ostream& VectorOps::printT(std::ostream &os, const std::vector<tp_Type> &v) {
	using namespace std;
	cout << "(";
	for (size_t i = 0; i < v.size(); ++i) cout << ((i > 0) ? ", " : "") << v[i];
	cout << ")";
	return cout;
}


template<typename tp_Type> void VectorOps::stuffT(std::vector<tp_Type> &stuffed, std::vector<int32_t> &sizes, const std::vector<tp_Type> &v) {
	size_t offset = stuffed.size();
	size_t requiredSize = offset + v.size();
	if (stuffed.capacity() < requiredSize)
		stuffed.reserve(std::max(stuffed.capacity() * 2, requiredSize));
	stuffed.resize(requiredSize);
	for (size_t i = 0; i < v.size(); ++i) stuffed[offset + i] = v[i];
	if (sizes.capacity() < sizes.size() + 1) sizes.reserve(std::max(sizes.capacity() * 2, sizes.size() + 1));
	sizes.push_back(v.size());
}


{'fraction_non_alphanumeric': 0.16434125039940356, 'fraction_numerical': 0.02098200021301523, 'mean_word_length': 6.09214501510574, 'pattern_counts': {'":': 0, '<': 171, '<?xml version=': 0, '>': 148, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1496165', 'n_tokens_mistral': 3522, 'n_tokens_neox': 3382, 'n_words': 942}