<commit_before>#include "encoder.h"

vsize_t* Encoder::encode(size_t value, size_t width=1) {
	vsize_t result = this->size_tToByteArray(value);
	vsize_t* val;
	if width < 4:
		val.= new vsize_t(4-width);
		for (size_t i = 0; i < val.size(); ++i) {
			(*val)[i] = result[i+4-width];
		}
	else {
		val = new vsize_t(width - 4 + result.size(), 0);
		for (size_t i = width - 4; i < val.size(); ++i) {
			val[i] = result[i-width - 4];
		}
	}
	return val;
}
size_t Encoder::decode(size_t value) {
	return this->byteArrayToSize_t(value);
}

vsize_t* Encoder::size_tToByteArray(size_t pValue) {
	vsize_t* byteArray = new vsize_t(4);
	(*byteArray)[0] = value >> 24 & 0x000000FF;
	(*byteArray)[1] = value >> 16 & 0x000000FF;
	(*byteArray)[2] = value >> 8 & 0x000000FF;
	(*byteArray)[3] = value >> 0 & 0x000000FF;
	return byteArray;
}

size_t Encoder::byteArrayToSize_t(vsize_t* pArray) {
	vsize_t* val;
	if (pArray->size() < 4) {
		val = new vsize_t(4, 0);
		for (size_t i = 4 - pArray->size(); i < val.size(); ++i) {
			val[i] = (*pArray)[i-pArray->size()];
		}
	} else {
		val = pArray;
	}
	return (*val)[0] << 24 + (*val)[1] << 16 + (*val)[2] << 8 + (*val)[3];
}<commit_msg>New constructor, new implementation of encode and decode, deleted others functions<commit_after>#include "encoder.h"

Encoder::Encoder(size_t pBitVectorSize) {
	mBitVectorSize = pBitVectorSize;
	mMaskingValues = new vsize_t(pBitVectorSize);
	size_t mask = 0b11111111;
	for (size_t i = 0; i < mBitVectorSize; ++i) {
		(*mMaskingValues)[i] = mask << (8*i);
	}
}
bitVector* Encoder::encode(size_t pValue) {
	bitVector value = new bitVector(mBitVectorSize);
	for (size_t i = 0; i < mBitVectorSize; ++i) {
		value[i] = pValue & mMaskingValues[i];
	}
	return value;
}
size_t Encoder::decode(bitVector* pValue) {
	size_t value = 0;
	size_t castValue = 0;
	for (size_t i = 0; i < pValue->size(); ++i) {
		castValue = (*pValue)[i];
		value = value | (castValue << (8*i));
	}
	return value;
}
{'fraction_non_alphanumeric': 0.18522319138019497, 'fraction_numerical': 0.03950743971267317, 'mean_word_length': 4.284552845528455, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1559824', 'n_tokens_mistral': 894, 'n_tokens_neox': 814, 'n_words': 185}