<commit_before>#pragma once
//HDF5:
#include "hdf5.h"
//#include "hdf5_hl.h"
//STL:
#include <utility>
#include <vector>
#include <array>
#include <iterator>
#include <numeric>
#include <functional>
#include <algorithm>
#include <stdexcept>
#include <sstream>

namespace H5TL {
	namespace util {
		template<typename InputIt, typename T>
		T product(InputIt first, InputIt last, T init) {
			return accumulate(first, last, init, multiplies<T>());
		}

		template<typename T>
		T clip(const T& val, const T& min, const T& max) {
			if (val < min) return min;
			else if (val > max) return max;
			else return val;
		}

		template<typename T>
		void prepend(std::vector<T> &vec, size_t N, const T& t) {
			std::vector<T> tmp(vec.size() + N);
			std::fill_n(tmp.begin(), N, t);
			std::copy_backward(vec.begin(), vec.end(), tmp.end());
			vec = std::move(tmp);
		}

		template<typename InputIt>
		std::string join(const std::string& delim, InputIt first, InputIt last) {
			ostringstream builder;
			builder << *first; ++first;
			while (first != last) {
				builder << delim << *first;
				++first;
			}
			return builder.str();
		}
	}
	//General library interface:

	///exception class for all HDF5 errors.
	class h5tl_error : public std::runtime_error {
	public:
		explicit h5tl_error(const std::string& what_arg) : std::runtime_error(what_arg) {}
		explicit h5tl_error(const char* what_arg) : std::runtime_error(what_arg) {}
	};

	///class for walking the HDF5 error stack.
	class ErrorHandler {
	protected:
		//use constructor of private static member to run this when the library loads:
		ErrorHandler() {
			//disable automatic error printing
			H5Eset_auto(H5E_DEFAULT, nullptr, nullptr);
		}
		static const ErrorHandler EH;

		static std::string class_name(hid_t class_id) {
			ssize_t len = H5Eget_class_name(class_id, nullptr, 0);
			std::string tmp(len, '\0');
			H5Eget_class_name(class_id, &(tmp[0]), len);
			return tmp;
		}
		static herr_t append_error(unsigned n, const H5E_error2_t *err_desc, void *client_data) {
			std::ostream &os = *(std::ostream*)client_data;
			os << "#" << n << ": " << err_desc->file_name << " line " << err_desc->line << ": " << err_desc->func_name << "(): " << std::endl;
			os << "    " << H5Eget_major(err_desc->maj_num) << ": " << H5Eget_minor(err_desc->min_num) << ": " << err_desc->desc << std::endl;
			//os << "class: " << _error_class_name(err_desc->cls_id) << std::endl;
			return 1;
		}
	public:
		/** \brief Return a std::string for the HDF5 error stack.

		Walks and then clears the indicated HDF5 error stack, building a string description of the current error.
		\param[in] es The error stack to walk, and subsequently clear.
		\return A string representation of the error stack.
		*/
		static std::string get_error(hid_t es = H5E_DEFAULT) {
			std::ostringstream oss;
			H5Ewalk2(es, H5E_WALK_DOWNWARD, append_error, &oss);
			H5Eclear2(es);
			return oss.str();
		}
	};
	const ErrorHandler ErrorHandler::EH;

	/** \brief Checks an HDF5 herr_t return code for errors.
	*
	* Checks an HDF5 herr_t return code for errors. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param[in] e Error code to check, throws an H5TL::h5tl_error if e < 0.
	*/
	inline void check(herr_t e) {
		if (e < 0) throw h5tl_error(ErrorHandler::get_error());
	}

	/** \brief Checks an HDF5 htri_t return code for errors and converts it to a bool.
	*
	* Checks an HDF5 tri_t return code for errors and converts it to a bool. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param t The htri_t return code to check
	* \returns true if t > 0
	*/
	inline bool check_tri(htri_t t) {
		if (t < 0) throw h5tl_error(ErrorHandler::get_error());
		else return t > 0;
	}
	/** \brief Checks an HDF5 hid_t return code for errors.
	*
	* Checks an HDF5 hid_t return code for errors and returns it. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param id The hid_t return code to check.
	* \returns id, unchanged.
	*/
	inline hid_t check_id(hid_t id) {
		if (id < 0) throw h5tl_error(ErrorHandler::get_error());
		else return id;
	}
	/** \brief Checks an HDF5 ssize_t return code for errors.
	*
	* Checks an HDF5 ssize_t return code for errors and returns it. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param sz The ssize_t return code to check.
	* \returns sz, unchanged.
	*/
	inline ssize_t check_size(ssize_t sz) {
		if (sz < 0) throw h5tl_error(ErrorHandler::get_error());
		else return sz;
	}
	/**
	Initializes the HDF5 library.
	*/
	inline void open() {
		check(H5open());
	}
	/**
	Close the HDF5 library.
	*/
	inline void close() {
		check(H5close());
	}

	class ID;
	void swap(ID&, ID&);

	/** \brief Virtual class encapsulating an hid_t. */
	class ID {
		friend void swap(ID&, ID&);
	protected:
		hid_t id;
		ID() : id(0) {}
		ID(hid_t _id) : id(check_id(_id)) {}
		//copy constructor is protected - can be made public as necessary by sub-classes
		ID(const ID& i) : id(i.id) {}
		//copy assign is protected - can be made public as necessary by sub-classes
		ID& operator=(const ID& i) {
			if (id) close();
			id = i.id;
			return *this;
		}
		void steal(ID& i) {
			if (id) close();
			id = i.id;
			i.id = 0;
		}
		void swap(ID& i1) {
			std::swap(id, i1.id);
		}
	public:
		//move constructor
		ID(ID &&i) : id(i.id) { i.id = 0; }
		
		//move assignment
		ID& operator=(ID&& i)  {
			steal(i);
			return *this;
		}

		virtual ~ID() {}
		virtual void close() = 0;
		virtual operator hid_t() const { return id; }
		virtual bool operator==(const ID& other) const {
			return this == &other || id == other.id;
		}
		/** \brief Checks that this ID refers to a valid HDF5 object.
		* \returns true if this refers to a valid HDF5 object.
		*/
		virtual bool valid() const { return check_tri(H5Iis_valid(id)); }
		virtual operator bool() const { return valid(); }
		
	};
	void swap(ID& i0, ID& i1) {
		i0.swap(i1);
	}

	class DType;
	class PDType;

	/** \brief Encapsulate an HDF5 data type object.
	*
	* DetailedDescription
	*/
	class DType : public ID {
		friend class Dataset;
		friend class Attribute;
	protected:
		DType(hid_t id) : ID(id) {}
		DType(hid_t id, size_t sz) : ID(H5Tcopy(id)) {
			size(sz);
		}
	public:
		
		//copy
		DType(const DType &dt) : ID(check_id(H5Tcopy(dt))) {}
		DType& operator=(DType dt) {<commit_msg>Minor docs<commit_after>#pragma once
//HDF5:
#include "hdf5.h"
//#include "hdf5_hl.h"
//STL:
#include <utility>
#include <vector>
#include <array>
#include <iterator>
#include <numeric>
#include <functional>
#include <algorithm>
#include <stdexcept>
#include <sstream>

namespace H5TL {
	namespace util {
		template<typename InputIt, typename T>
		T product(InputIt first, InputIt last, T init) {
			return accumulate(first, last, init, multiplies<T>());
		}

		template<typename T>
		T clip(const T& val, const T& min, const T& max) {
			if (val < min) return min;
			else if (val > max) return max;
			else return val;
		}

		template<typename T>
		void prepend(std::vector<T> &vec, size_t N, const T& t) {
			std::vector<T> tmp(vec.size() + N);
			std::fill_n(tmp.begin(), N, t);
			std::copy_backward(vec.begin(), vec.end(), tmp.end());
			vec = std::move(tmp);
		}

		template<typename InputIt>
		std::string join(const std::string& delim, InputIt first, InputIt last) {
			ostringstream builder;
			builder << *first; ++first;
			while (first != last) {
				builder << delim << *first;
				++first;
			}
			return builder.str();
		}
	}
	//General library interface:

	///exception class for all HDF5 errors.
	class h5tl_error : public std::runtime_error {
	public:
		explicit h5tl_error(const std::string& what_arg) : std::runtime_error(what_arg) {}
		explicit h5tl_error(const char* what_arg) : std::runtime_error(what_arg) {}
	};

	///class for walking the HDF5 error stack.
	class ErrorHandler {
	protected:
		//use constructor of private static member to run this when the library loads:
		ErrorHandler() {
			//disable automatic error printing
			H5Eset_auto(H5E_DEFAULT, nullptr, nullptr);
		}
		static const ErrorHandler EH;

		static std::string class_name(hid_t class_id) {
			ssize_t len = H5Eget_class_name(class_id, nullptr, 0);
			std::string tmp(len, '\0');
			H5Eget_class_name(class_id, &(tmp[0]), len);
			return tmp;
		}
		static herr_t append_error(unsigned n, const H5E_error2_t *err_desc, void *client_data) {
			std::ostream &os = *(std::ostream*)client_data;
			os << "#" << n << ": " << err_desc->file_name << " line " << err_desc->line << ": " << err_desc->func_name << "(): " << std::endl;
			os << "    " << H5Eget_major(err_desc->maj_num) << ": " << H5Eget_minor(err_desc->min_num) << ": " << err_desc->desc << std::endl;
			//os << "class: " << _error_class_name(err_desc->cls_id) << std::endl;
			return 1;
		}
	public:
		/** \brief Return a std::string for the HDF5 error stack.

		Walks and then clears the indicated HDF5 error stack, building a string description of the current error.
		\param[in] es The error stack to walk, and subsequently clear.
		\return A string representation of the error stack.
		*/
		static std::string get_error(hid_t es = H5E_DEFAULT) {
			std::ostringstream oss;
			H5Ewalk2(es, H5E_WALK_DOWNWARD, append_error, &oss);
			H5Eclear2(es);
			return oss.str();
		}
	};
	const ErrorHandler ErrorHandler::EH;

	/** \brief Checks an HDF5 herr_t return code for errors.
	*
	* Checks an HDF5 herr_t return code for errors. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param[in] e Error code to check, throws an H5TL::h5tl_error if e < 0.
	*/
	inline void check(herr_t e) {
		if (e < 0) throw h5tl_error(ErrorHandler::get_error());
	}

	/** \brief Checks an HDF5 htri_t return code for errors and converts it to a bool.
	*
	* Checks an HDF5 tri_t return code for errors and converts it to a bool. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param t The htri_t return code to check
	* \returns true if t > 0
	*/
	inline bool check_tri(htri_t t) {
		if (t < 0) throw h5tl_error(ErrorHandler::get_error());
		else return t > 0;
	}
	/** \brief Checks an HDF5 hid_t return code for errors.
	*
	* Checks an HDF5 hid_t return code for errors and returns it. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param id The hid_t return code to check.
	* \returns id, unchanged.
	*/
	inline hid_t check_id(hid_t id) {
		if (id < 0) throw h5tl_error(ErrorHandler::get_error());
		else return id;
	}
	/** \brief Checks an HDF5 ssize_t return code for errors.
	*
	* Checks an HDF5 ssize_t return code for errors and returns it. If there is an error, throws an H5TL::h5tl_error with the message given by ErrorHandler::get_error().
	* \param sz The ssize_t return code to check.
	* \returns sz, unchanged.
	*/
	inline ssize_t check_size(ssize_t sz) {
		if (sz < 0) throw h5tl_error(ErrorHandler::get_error());
		else return sz;
	}
	/**
	Initializes the HDF5 library.
	*/
	inline void open() {
		check(H5open());
	}
	/**
	Close the HDF5 library.
	*/
	inline void close() {
		check(H5close());
	}

	class ID;
	void swap(ID&, ID&);

	/** \brief Virtual class encapsulating an hid_t. */
	class ID {
		friend void swap(ID&, ID&);
	protected:
		hid_t id;
		ID() : id(0) {}
		ID(hid_t _id) : id(check_id(_id)) {}
		//copy constructor is protected - can be used as necessary by sub-classes
		ID(const ID& i) : id(i.id) {}
		//copy assign is protected - can be used as necessary by sub-classes
		ID& operator=(const ID& i) {
			if (id) close();
			id = i.id;
			return *this;
		}
		//steal the underlying resources from another ID (calls close() first if necessary)
		void steal(ID& i) {
			if (id) close();
			id = i.id;
			i.id = 0;
		}
		//swap underlying resources with another ID
		void swap(ID& i1) {
			std::swap(id, i1.id);
		}
	public:
		//move constructor
		ID(ID &&i) : id(i.id) { i.id = 0; }
		
		//move assignment
		ID& operator=(ID&& i)  {
			steal(i);
			return *this;
		}

		virtual ~ID() {}

		//close the ID
		virtual void close() = 0;

		//convert to hid_t, so that we can use H5TL objects with the corresponding C library
		virtual operator hid_t() const { return id; }
		virtual bool operator==(const ID& other) const {
			return this == &other || id == other.id;
		}
		/** \brief Checks that this ID refers to a valid HDF5 object.
		* \returns true if this refers to a valid HDF5 object.
		*/
		virtual bool valid() const { return check_tri(H5Iis_valid(id)); }
		virtual operator bool() const { return valid(); }
		
	};

	//swap 2 IDs -- should this be in std?
	void swap(ID& i0, ID& i1) {
		i0.swap(i1);
	}

	class DType;
	class PDType;

	/** \brief Encapsulate an HDF5 data type object.
	*
	* DetailedDescription
	*/
	class DType : public ID {
		friend class Dataset;
		friend class Attribute;
	protected:
		DType(hid_t id) : ID(id) {}
		DType(hid_t id, size_t sz) : ID(H5Tcopy(id)) {
			size(sz);
		}
	public:
		
		//copy
		DType(const DType &dt) : ID(check_id(H5Tcopy(dt))) {}
		DType& operator=(DType dt) {
{'fraction_non_alphanumeric': 0.13553943789664552, 'fraction_numerical': 0.011937141130250831, 'mean_word_length': 3.5534915720674234, 'pattern_counts': {'":': 8, '<': 139, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2585138', 'n_tokens_mistral': 5193, 'n_tokens_neox': 4779, 'n_words': 1511}