<commit_before>/*
 *   CrissCross
 *   A multi-purpose cross-platform library.
 *
 *   A product of Uplink Laboratories.
 *
 *   (c) 2006-2021 <NAME>.
 *   Licensed under the New BSD License.
 *
 */

#ifndef __included_cc_darray_h
#error "This file shouldn't be compiled directly."
#endif

#include <crisscross/debug.h>

#include <crisscross/darray.h>
#include <crisscross/dstack.h>

namespace CrissCross
{
	namespace Data
	{
		template <class T>
		DArray <T>::DArray()
		{
			static_assert(std::is_trivially_copyable<T>::value);
			m_stepSize = -1;
			m_numUsed = m_arraySize = 0;
			m_array = NULL;
		}

		template <class T>
		DArray <T>::DArray(T *_array, size_t _indices)
		{
			m_array = new T[_indices];
			memcpy(m_array, _array, _indices * sizeof(T));

			m_shadow.resize(_indices, false);

			m_numUsed = 0;
			for (size_t i = 0; i < _indices; i++) {
				m_shadow[i] = (m_array[i] != NULL) ? true : false;
				if (m_shadow[i]) {
					m_numUsed++;
				}
			}

			m_arraySize = _indices;
			m_stepSize = -1;
		}

		template <class T>
		DArray <T>::DArray(DArray<T> const &_array)
		{
			m_array = new T[_array.m_arraySize];
			memcpy(m_array, _array.m_array, _array.m_arraySize * sizeof(T));

			m_shadow.resize(_array.m_arraySize);

			m_numUsed = 0;
			for (size_t i = 0; i < _array.m_arraySize; i++)	{
				m_shadow[i] = (m_array[i] != NULL) ? true : false;
				if (m_shadow[i]) {
					m_numUsed++;
				}
			}

			m_arraySize = _array.m_arraySize;
			m_stepSize = _array.m_stepSize;
		}

		template <class T>
		DArray <T>::DArray(int _newStepSize)
		{
			if (_newStepSize < 1)
				m_stepSize = -1;
			else
				m_stepSize = _newStepSize;

			m_numUsed = m_arraySize = 0;
			m_array = NULL;
		}

		template <class T>
		DArray <T>::~DArray()
		{
			empty();
		}

		template <class T>
		void DArray <T>::compact()
		{
			constexpr size_t pageSizeInElements = std::max((size_t)(4096u / sizeof(T)), (size_t)32u);

			/* Only resize if it would possibly be beneficial. */
			if ((m_arraySize * sizeof(T)) <= 4096)
				return;
			if (m_numUsed > m_arraySize / 2)
				return;

			auto found = std::find(std::rbegin(m_shadow), std::rend(m_shadow), 1);
			size_t idx = std::distance(found, m_shadow.rend());

			if (idx > m_arraySize)
				return;

			if ((m_arraySize - idx) < pageSizeInElements * 2)
				return;
			
			/* Only shrink if the array is significantly under-used */
			float inUse = (float)idx / (float)m_arraySize;
			if (inUse < 0.33f) {
				/* Round up to next page boundary. */
				idx = ((idx / pageSizeInElements) + 1) * pageSizeInElements;
				setSize(idx);
			}
		}

		template <class T>
		void DArray <T>::recount()
		{
			m_numUsed = 0;
			for (size_t i = 0; i < m_arraySize; i++)
				if (m_shadow[i])
					m_numUsed++;
		}

		template <class T>
		void DArray <T>::reserve(size_t newsize)
		{
			setSize(newsize);
			m_shadow.clear();
			m_shadow.resize(newsize, true);
		}

		template <class T>
		void DArray <T>::setMinimumSize(size_t newsize)
		{
			if (m_arraySize < newsize)
				setSize(newsize);
		}

		template <class T>
		void DArray <T>::setSize(size_t newsize)
		{
			m_shadow.resize(newsize, false);
			if (newsize > m_arraySize) {
				size_t oldarraysize = m_arraySize;

				m_arraySize = newsize;
				T *temparray = new T[m_arraySize];
<commit_msg>darray: treat ::reserve() like a better std::vector::reserve<commit_after>/*
 *   CrissCross
 *   A multi-purpose cross-platform library.
 *
 *   A product of Uplink Laboratories.
 *
 *   (c) 2006-2021 <NAME>.
 *   Licensed under the New BSD License.
 *
 */

#ifndef __included_cc_darray_h
#error "This file shouldn't be compiled directly."
#endif

#include <crisscross/debug.h>

#include <crisscross/darray.h>
#include <crisscross/dstack.h>

namespace CrissCross
{
	namespace Data
	{
		template <class T>
		DArray <T>::DArray()
		{
			static_assert(std::is_trivially_copyable<T>::value);
			m_stepSize = -1;
			m_numUsed = m_arraySize = 0;
			m_array = NULL;
		}

		template <class T>
		DArray <T>::DArray(T *_array, size_t _indices)
		{
			m_array = new T[_indices];
			memcpy(m_array, _array, _indices * sizeof(T));

			m_shadow.resize(_indices, false);

			m_numUsed = 0;
			for (size_t i = 0; i < _indices; i++) {
				m_shadow[i] = (m_array[i] != NULL) ? true : false;
				if (m_shadow[i]) {
					m_numUsed++;
				}
			}

			m_arraySize = _indices;
			m_stepSize = -1;
		}

		template <class T>
		DArray <T>::DArray(DArray<T> const &_array)
		{
			m_array = new T[_array.m_arraySize];
			memcpy(m_array, _array.m_array, _array.m_arraySize * sizeof(T));

			m_shadow.resize(_array.m_arraySize);

			m_numUsed = 0;
			for (size_t i = 0; i < _array.m_arraySize; i++)	{
				m_shadow[i] = (m_array[i] != NULL) ? true : false;
				if (m_shadow[i]) {
					m_numUsed++;
				}
			}

			m_arraySize = _array.m_arraySize;
			m_stepSize = _array.m_stepSize;
		}

		template <class T>
		DArray <T>::DArray(int _newStepSize)
		{
			if (_newStepSize < 1)
				m_stepSize = -1;
			else
				m_stepSize = _newStepSize;

			m_numUsed = m_arraySize = 0;
			m_array = NULL;
		}

		template <class T>
		DArray <T>::~DArray()
		{
			empty();
		}

		template <class T>
		void DArray <T>::compact()
		{
			constexpr size_t pageSizeInElements = std::max((size_t)(4096u / sizeof(T)), (size_t)32u);

			/* Only resize if it would possibly be beneficial. */
			if ((m_arraySize * sizeof(T)) <= 4096)
				return;
			if (m_numUsed > m_arraySize / 2)
				return;

			auto found = std::find(std::rbegin(m_shadow), std::rend(m_shadow), 1);
			size_t idx = std::distance(found, m_shadow.rend());

			if (idx > m_arraySize)
				return;

			if ((m_arraySize - idx) < pageSizeInElements * 2)
				return;
			
			/* Only shrink if the array is significantly under-used */
			float inUse = (float)idx / (float)m_arraySize;
			if (inUse < 0.33f) {
				/* Round up to next page boundary. */
				idx = ((idx / pageSizeInElements) + 1) * pageSizeInElements;
				setSize(idx);
			}
		}

		template <class T>
		void DArray <T>::recount()
		{
			m_numUsed = 0;
			for (size_t i = 0; i < m_arraySize; i++)
				if (m_shadow[i])
					m_numUsed++;
		}

		template <class T>
		void DArray <T>::reserve(size_t newsize)
		{
			if (newsize > m_arraySize) {
				setSize(newsize);
				m_shadow.clear();
				m_shadow.resize(newsize, true);
			}
		}

		template <class T>
		void DArray <T>::setMinimumSize(size_t newsize)
		{
			if (m_arraySize < newsize)
				setSize(newsize);
		}

		template <class T>
		void DArray <T>::setSize(size_t newsize)
		{
			m_shadow.resize(newsize, false);
			if (newsize > m_arraySize) {
				size_t oldarraysize = m_arraySize;

				m_arraySize = newsize;
				T *temparray = new T[m_arraySize];

{'fraction_non_alphanumeric': 0.14193354382799578, 'fraction_numerical': 0.011126146444143737, 'mean_word_length': 3.1011097410604194, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '898354', 'n_tokens_mistral': 3118, 'n_tokens_neox': 2586, 'n_words': 587}