<commit_before>#pragma once

#include <vector>
#include <string>
#include <string.h>
#include <assert.h>

namespace logging {

std::string byteArrayToString(const void* buffer, size_t length);

/**
 * A vector-like class which uses a statically allocated buffer if the data is small enough.
 */
class ByteArray {

public:
	ByteArray() {
#ifndef NDEBUG
		memset( m_staticData, 0, sizeof(m_staticData) );
#endif
	}

	ByteArray(const ByteArray& b) {
		// duplicate content
		append( b.getData(), b.size() );
	}

	ByteArray& operator=(const ByteArray& right) {
		resize( right.size() );
		memcpy( getData(), right.getData(), right.size() );
		return *this;
	}

	~ByteArray() {
		if ( !usesStaticBuffer() )
			delete (m_dynamicData);
	}

	size_t size() const {
		return ( usesStaticBuffer() ? m_length : m_dynamicData->size() );
	}

	bool usesStaticBuffer() const {
		return (m_dynamicData == nullptr);
	}

	char* getData() {
		if ( usesStaticBuffer() )
			return m_staticData;
		else
			return &( (*m_dynamicData)[0] );
	}

	const char* getData() const {
		if ( usesStaticBuffer() )
			return m_staticData;
		else
			return &( (*m_dynamicData)[0] );
	}

	char& operator[](size_t index) {
		assert(index < m_length);
		return getData()[index];
	}

	void resize(size_t size) {

		if ( !usesStaticBuffer() ) {
			m_dynamicData->resize(size);
		} else {
			if ( size <= sizeof(m_staticData) ) {
				m_length = size;
			} else {
				m_dynamicData = new std::vector<char>();
				m_dynamicData->resize(size);
				memcpy(m_dynamicData->data(), m_staticData, m_length);
				m_length = -1; // this field is not relevant anymore
			}
		}

	}

	void writeAt(size_t position, const void* rawDataPtr, size_t sizeInByte) {
		assert(m_length >= position + sizeInByte);
		memcpy(&(getData()[position]), rawDataPtr, sizeInByte);
	}

	void append(const void* rawDataPtr, size_t sizeInByte) {
		auto i = size();
		resize(size() + sizeInByte);
		memcpy(getData() + i, rawDataPtr, sizeInByte);
	}

	void append(unsigned char v) {
		auto i = size();
		resize(i + 1);
		getData()[i] = v;
	}

	size_t capacity() const {
		if ( usesStaticBuffer() )
			return sizeof(m_staticData);
		else
			return m_dynamicData->size();
	}

	/**
	 * Reserve space for a block of the given size and return the position of the block
	 */
	size_t skip(size_t length) {
		auto returnValue = size();
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wvla"
		char dummyData[length];
#pragma GCC diagnostic pop
		memset(dummyData, 0, length);
		append( dummyData, sizeof(dummyData) );
		return returnValue;
	}

	std::string toString() const {
		return byteArrayToString( getData(), size() );
	}

private:
	std::vector<char>* m_dynamicData = nullptr;
	char m_staticData[512];
	size_t m_length = 0;
};

typedef ByteArray ResizeableByteArray;

}<commit_msg>Remove unused method "skip()" (#12)<commit_after>#pragma once

#include <vector>
#include <string>
#include <string.h>
#include <assert.h>

namespace logging {

std::string byteArrayToString(const void* buffer, size_t length);

/**
 * A vector-like class which uses a statically allocated buffer if the data is small enough.
 */
class ByteArray {

public:
	ByteArray() {
#ifndef NDEBUG
		memset( m_staticData, 0, sizeof(m_staticData) );
#endif
	}

	ByteArray(const ByteArray& b) {
		// duplicate content
		append( b.getData(), b.size() );
	}

	ByteArray& operator=(const ByteArray& right) {
		resize( right.size() );
		memcpy( getData(), right.getData(), right.size() );
		return *this;
	}

	~ByteArray() {
		if ( !usesStaticBuffer() )
			delete (m_dynamicData);
	}

	size_t size() const {
		return ( usesStaticBuffer() ? m_length : m_dynamicData->size() );
	}

	bool usesStaticBuffer() const {
		return (m_dynamicData == nullptr);
	}

	char* getData() {
		if ( usesStaticBuffer() )
			return m_staticData;
		else
			return &( (*m_dynamicData)[0] );
	}

	const char* getData() const {
		if ( usesStaticBuffer() )
			return m_staticData;
		else
			return &( (*m_dynamicData)[0] );
	}

	char& operator[](size_t index) {
		assert(index < m_length);
		return getData()[index];
	}

	void resize(size_t size) {

		if ( !usesStaticBuffer() ) {
			m_dynamicData->resize(size);
		} else {
			if ( size <= sizeof(m_staticData) ) {
				m_length = size;
			} else {
				m_dynamicData = new std::vector<char>();
				m_dynamicData->resize(size);
				memcpy(m_dynamicData->data(), m_staticData, m_length);
				m_length = -1; // this field is not relevant anymore
			}
		}

	}

	void writeAt(size_t position, const void* rawDataPtr, size_t sizeInByte) {
		assert(m_length >= position + sizeInByte);
		memcpy(&(getData()[position]), rawDataPtr, sizeInByte);
	}

	void append(const void* rawDataPtr, size_t sizeInByte) {
		auto i = size();
		resize(size() + sizeInByte);
		memcpy(getData() + i, rawDataPtr, sizeInByte);
	}

	void append(unsigned char v) {
		auto i = size();
		resize(i + 1);
		getData()[i] = v;
	}

	size_t capacity() const {
		if ( usesStaticBuffer() )
			return sizeof(m_staticData);
		else
			return m_dynamicData->size();
	}

	std::string toString() const {
		return byteArrayToString( getData(), size() );
	}

private:
	std::vector<char>* m_dynamicData = nullptr;
	char m_staticData[512];
	size_t m_length = 0;
};

typedef ByteArray ResizeableByteArray;

}
{'fraction_non_alphanumeric': 0.1430197268588771, 'fraction_numerical': 0.003983308042488619, 'mean_word_length': 4.041108986615678, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5972955', 'n_tokens_mistral': 1996, 'n_tokens_neox': 1817, 'n_words': 461}