<commit_before>#include <cassert>

#include "Tools/Display/bash_tools.h"
#include "Tools/Algo/Bit_packer.hpp"

#include "CRC_polynomial_fast.hpp"

template <typename B>
CRC_polynomial_fast<B>
::CRC_polynomial_fast(const int K, std::string poly_key, const int size, const int n_frames, const std::string name)
: CRC_polynomial<B>(K, poly_key, size, n_frames, name), lut_crc32(256), polynomial_packed_rev(0)
{
	assert(this->size() <= 32);

	// reverse the order of the bits in the bitpacked polynomial
	for (auto i = 0; i < (int)sizeof(this->polynomial_packed) * 8; i++)
	{
		polynomial_packed_rev <<= 1;
		polynomial_packed_rev |= (this->polynomial_packed >> i) & 1;
	}
	polynomial_packed_rev >>= (sizeof(this->polynomial_packed) * 8) - (this->size());

	// precompute a lookup table pour the v2 implem. of the CRC
	for (auto i = 0; i < 256; i++)
	{
		unsigned crc = i;
		for (unsigned int j = 0; j < 8; j++)
			crc = (crc >> 1) ^ (-int(crc & 1) & polynomial_packed_rev);
		lut_crc32[i] = crc;
	}
}

template <typename B>
void CRC_polynomial_fast<B>
::build(mipp::vector<B>& U_K)
{
	assert(U_K.size() >  (unsigned)(this->n_frames * this->size()));
	assert(U_K.size() == (unsigned)(this->n_frames * this->K));

	Bit_packer<B>::pack(U_K, this->buff_crc, this->n_frames);

	const auto n_bits_per_frame  = this->K;
	const auto n_bytes_per_frame = static_cast<int>(std::ceil((float)n_bits_per_frame / 8.f));

	auto bytes = (unsigned char*)this->buff_crc.data();
	for (auto f = 0; f < this->n_frames; f++)
	{
		auto data = bytes + f * n_bytes_per_frame;
		const auto crc  = this->compute_crc_v2((void*)data, n_bits_per_frame - this->size());

		for (auto i = 0; i < this->size(); i++)
			U_K[this->K * (f +1) - this->size() +i] = (crc >> i) & 1;
	}
}

template <typename B>
bool CRC_polynomial_fast<B>
::check(const mipp::vector<B>& V_K, const int n_frames)
{
	const int real_n_frames = (n_frames != -1) ? n_frames : this->n_frames;

	assert(V_K.size() == this->buff_crc.size());
	Bit_packer<B>::pack(V_K, this->buff_crc, real_n_frames);
	return this->check_packed(this->buff_crc, n_frames);
}

template <typename B>
bool CRC_polynomial_fast<B>
::check_packed(const mipp::vector<B>& V_K, const int n_frames)
{
#if __BYTE_ORDER != __LITTLE_ENDIAN
	std::cout << bold_red("(EE) This fast CRC code works only on little endian CPUs (x86, ARM, ...).") << std::endl;
	std::exit(-1);
#endif

	const int real_n_frames = (n_frames != -1) ? n_frames : this->n_frames;
	assert(real_n_frames > 0);
	assert((int)V_K.size() % real_n_frames == 0);

	const auto crc_size          = this->size();
	const auto n_bits_per_frame  = (int)((int)V_K.size() / real_n_frames) - this->size();
	const auto n_bytes_per_frame = static_cast<int>(std::ceil((float)n_bits_per_frame / 8.f));
	const auto rest              = n_bits_per_frame % 8;

	unsigned char* bytes = (unsigned char*)V_K.data();

	unsigned crc_invalid = 0;
	for (auto f = 0; f < real_n_frames && !crc_invalid; f++)
	{
		const auto data = bytes + f * n_bytes_per_frame;
		const auto crc  = this->compute_crc_v2((void*)data, n_bits_per_frame);

		auto n_bits_crc = crc_size;
		auto current = data + (n_bits_per_frame / 8);

		unsigned crc_ref = 0;
		if (rest)
		{
			auto cur = *current++;
			for (auto j = rest; j < 8 && n_bits_crc > 0; j++)
				crc_ref |= ((cur >> j) & 1) << (crc_size - n_bits_crc--);
		}
		while (n_bits_crc)
		{
			auto cur = *current++;
			for (auto j = 0; j < 8 && n_bits_crc > 0; j++)
				crc_ref |= ((cur >> j) & 1) << (crc_size - n_bits_crc--);
		}

		crc_invalid |= crc ^ crc_ref;
	}

	return !crc_invalid;
}

// Source of inspiration: http://create.stephan-brumme.com/crc32/ (Fastest Bitwise CRC32)
template <typename B>
unsigned CRC_polynomial_fast<B>
::compute_crc_v1(const void* data, const int n_bits)
{
	unsigned crc = 0;

	auto current = (unsigned char*)data;
	auto length = n_bits / 8;
	while (length--)
	{
		crc ^= *current++;
		for (auto j = 0; j < 8; j++)
//			crc = (crc & 1) ? (crc >> 1) ^ polynomial_packed_rev : crc >> 1;
			crc = (crc >> 1) ^ (-int(crc & 1) & polynomial_packed_rev);
	}

	auto rest = n_bits % 8;
	if (rest != 0)
	{
		auto cur = *current;
		cur <<= 8 - rest;
		cur >>= 8 - rest;

		crc ^= cur;
		for (auto j = 0; j < rest; j++)
//			crc = (crc & 1) ? (crc >> 1) ^ polynomial_packed_rev : crc >> 1;
			crc = (crc >> 1) ^ (-int(crc & 1) & polynomial_packed_rev);
	}

	return crc;
}

// Source of inspiration: http://create.stephan-brumme.com/crc32/ (Standard Implementation)
template <typename B>
unsigned CRC_polynomial_fast<B>
::compute_crc_v2(const void* data, const int n_bits)
{
	unsigned crc = 0;

	auto current = (unsigned char*)data;
	auto length = n_bits / 8;
	while (length--)
		crc = (crc >> 8) ^ lut_crc32[(crc & 0xFF) ^ *current++];

	auto rest = n_bits % 8;
	if (rest != 0)
	{
		auto cur = *current;<commit_msg>Add little endian warnings.<commit_after>#include <cassert>

#include "Tools/Display/bash_tools.h"
#include "Tools/Algo/Bit_packer.hpp"

#include "CRC_polynomial_fast.hpp"

template <typename B>
CRC_polynomial_fast<B>
::CRC_polynomial_fast(const int K, std::string poly_key, const int size, const int n_frames, const std::string name)
: CRC_polynomial<B>(K, poly_key, size, n_frames, name), lut_crc32(256), polynomial_packed_rev(0)
{
	assert(this->size() <= 32);

	// reverse the order of the bits in the bitpacked polynomial
	for (auto i = 0; i < (int)sizeof(this->polynomial_packed) * 8; i++)
	{
		polynomial_packed_rev <<= 1;
		polynomial_packed_rev |= (this->polynomial_packed >> i) & 1;
	}
	polynomial_packed_rev >>= (sizeof(this->polynomial_packed) * 8) - (this->size());

	// precompute a lookup table pour the v2 implem. of the CRC
	for (auto i = 0; i < 256; i++)
	{
		unsigned crc = i;
		for (unsigned int j = 0; j < 8; j++)
			crc = (crc >> 1) ^ (-int(crc & 1) & polynomial_packed_rev);
		lut_crc32[i] = crc;
	}
}

template <typename B>
void CRC_polynomial_fast<B>
::build(mipp::vector<B>& U_K)
{
#if __BYTE_ORDER != __LITTLE_ENDIAN
	std::cout << bold_red("(EE) This fast CRC code works only on little endian CPUs (x86, ARM, ...).") << std::endl;
	std::exit(-1);
#endif

	assert(U_K.size() >  (unsigned)(this->n_frames * this->size()));
	assert(U_K.size() == (unsigned)(this->n_frames * this->K));

	Bit_packer<B>::pack(U_K, this->buff_crc, this->n_frames);

	const auto n_bits_per_frame  = this->K;
	const auto n_bytes_per_frame = static_cast<int>(std::ceil((float)n_bits_per_frame / 8.f));

	auto bytes = (unsigned char*)this->buff_crc.data();
	for (auto f = 0; f < this->n_frames; f++)
	{
		auto data = bytes + f * n_bytes_per_frame;
		const auto crc  = this->compute_crc_v2((void*)data, n_bits_per_frame - this->size());

		for (auto i = 0; i < this->size(); i++)
			U_K[this->K * (f +1) - this->size() +i] = (crc >> i) & 1;
	}
}

template <typename B>
bool CRC_polynomial_fast<B>
::check(const mipp::vector<B>& V_K, const int n_frames)
{
	const int real_n_frames = (n_frames != -1) ? n_frames : this->n_frames;

	assert(V_K.size() == this->buff_crc.size());
	Bit_packer<B>::pack(V_K, this->buff_crc, real_n_frames);
	return this->check_packed(this->buff_crc, n_frames);
}

template <typename B>
bool CRC_polynomial_fast<B>
::check_packed(const mipp::vector<B>& V_K, const int n_frames)
{
#if __BYTE_ORDER != __LITTLE_ENDIAN
	std::cout << bold_red("(EE) This fast CRC code works only on little endian CPUs (x86, ARM, ...).") << std::endl;
	std::exit(-1);
#endif

	const int real_n_frames = (n_frames != -1) ? n_frames : this->n_frames;
	assert(real_n_frames > 0);
	assert((int)V_K.size() % real_n_frames == 0);

	const auto crc_size          = this->size();
	const auto n_bits_per_frame  = (int)((int)V_K.size() / real_n_frames) - this->size();
	const auto n_bytes_per_frame = static_cast<int>(std::ceil((float)n_bits_per_frame / 8.f));
	const auto rest              = n_bits_per_frame % 8;

	unsigned char* bytes = (unsigned char*)V_K.data();

	unsigned crc_invalid = 0;
	for (auto f = 0; f < real_n_frames && !crc_invalid; f++)
	{
		const auto data = bytes + f * n_bytes_per_frame;
		const auto crc  = this->compute_crc_v2((void*)data, n_bits_per_frame);

		auto n_bits_crc = crc_size;
		auto current = data + (n_bits_per_frame / 8);

		unsigned crc_ref = 0;
		if (rest)
		{
			auto cur = *current++;
			for (auto j = rest; j < 8 && n_bits_crc > 0; j++)
				crc_ref |= ((cur >> j) & 1) << (crc_size - n_bits_crc--);
		}
		while (n_bits_crc)
		{
			auto cur = *current++;
			for (auto j = 0; j < 8 && n_bits_crc > 0; j++)
				crc_ref |= ((cur >> j) & 1) << (crc_size - n_bits_crc--);
		}

		crc_invalid |= crc ^ crc_ref;
	}

	return !crc_invalid;
}

// Source of inspiration: http://create.stephan-brumme.com/crc32/ (Fastest Bitwise CRC32)
template <typename B>
unsigned CRC_polynomial_fast<B>
::compute_crc_v1(const void* data, const int n_bits)
{
#if __BYTE_ORDER != __LITTLE_ENDIAN
	std::cout << bold_red("(EE) This fast CRC code works only on little endian CPUs (x86, ARM, ...).") << std::endl;
	std::exit(-1);
#endif

	unsigned crc = 0;

	auto current = (unsigned char*)data;
	auto length = n_bits / 8;
	while (length--)
	{
		crc ^= *current++;
		for (auto j = 0; j < 8; j++)
//			crc = (crc & 1) ? (crc >> 1) ^ polynomial_packed_rev : crc >> 1;
			crc = (crc >> 1) ^ (-int(crc & 1) & polynomial_packed_rev);
	}

	auto rest = n_bits % 8;
	if (rest != 0)
	{
		auto cur = *current;
		cur <<= 8 - rest;
		cur >>= 8 - rest;

		crc ^= cur;
		for (auto j = 0; j < rest; j++)
//			crc = (crc & 1) ? (crc >> 1) ^ polynomial_packed_rev : crc >> 1;
			crc = (crc >> 1) ^ (-int(crc & 1) & polynomial_packed_rev);
	}

	return crc;
}

// Source of inspiration: http://create.stephan-brumme.com/crc32/ (Standard Implementation)
template <typename B>
unsigned CRC_polynomial_fast<B>
::compute_crc_v2(const void* data, const int n_bits)
{
#if __BYTE_ORDER != __LITTLE_ENDIAN
	std::cout << bold_red("(EE) This fast CRC code works only on little endian CPUs (x86, ARM, ...).") << std::endl;
	std::exit(-1);
#endif

	unsigned crc = 0;

	auto current = (unsigned char*)data;
	auto length = n_bits / 8;
	while (length--)
		crc = (crc >> 8) ^ lut_crc32[(crc & 0xFF) ^ *current++];

	auto rest = n_bits % 8;
	if (rest != 0)
	{
		auto cur = *current;
{'fraction_non_alphanumeric': 0.17270578968097677, 'fraction_numerical': 0.01782197715636077, 'mean_word_length': 3.937773456490034, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 165, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6980529', 'n_tokens_mistral': 4416, 'n_tokens_neox': 4107, 'n_words': 1012}