<commit_before>#include "utility/crc.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>

#include <cstdio>
#include <iostream>
#include <thread>
#include <stack>
#include <random>

#include "cppunit-header.h"


template<class T, T Polynomial>
class Test_Crc : public CppUnit::TestFixture
{
	using calc_type = Crc<T, Polynomial>;

	using crc_type = T;

	CPPUNIT_TEST_SUITE(Test_Crc);
	CPPUNIT_TEST(testThreadedCreation);
	CPPUNIT_TEST(crcCalculation);
	CPPUNIT_TEST(crcFile<uint8_t>);
	CPPUNIT_TEST(crcFile<uint16_t>);
	CPPUNIT_TEST(crcFile<uint64_t>);
	CPPUNIT_TEST(crcFile<uint64_t>);
	CPPUNIT_TEST_SUITE_END();

 public:
	~Test_Crc()
	{
		unlink("data");
	}

	void setUp()
	{
		const size_t filesize = (1 << 24);
		uint64_t x;
		struct stat st;
		int rc, size = 0;;

		// force deterministic sequence
		std::mt19937_64 generator(0);
		std::uniform_int_distribution<uint64_t> dist(
			std::numeric_limits<uint64_t>::min(),
			std::numeric_limits<uint64_t>::max());

		if ( stat("data", &st) != 0 )
		{
			if ((randomData = open("data", O_CREAT|O_RDWR| O_TRUNC, 0644)) < 0)
				throw std::runtime_error("Fail");

			for (size_t i = 0; i < filesize/sizeof(x); ++i)
			{
				x = dist(generator);
				rc = write(randomData, reinterpret_cast<char*>(&x), sizeof(x));

				if (rc != sizeof(x))
					throw std::runtime_error("Write failed");

				size+=rc;
			}

			// printf("data created with size %d\n", size);
		} else
			if ((randomData = open("data", O_RDWR )) < 0)
				throw std::runtime_error("Fail");

		lseek(randomData, 0, SEEK_SET);
	}

	void tearDown() {
		close(randomData);
	}

	void testThreadedCreation()
	{
		std::vector<std::thread> threads;
		for (int i = 0; i < 100; ++i)
		{
			threads.emplace_back([] {
				Crc<T, Polynomial> c;
			});
		}

		for (auto & i : threads) i.join();
	}

	void crcCalculation()
	{
		calc_type crc1;
		crc1(datum1.c_str(), datum1.size());
		CPPUNIT_ASSERT(crc1.get() == datum1crc);

		calc_type crc2;
		crc2(datum2.c_str(), datum2.size());
		CPPUNIT_ASSERT(crc2.get() == datum2crc);<commit_msg>Moved unlink command to properly clean up stuff<commit_after>#include "utility/crc.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>

#include <cstdio>
#include <iostream>
#include <thread>
#include <stack>
#include <random>

#include "cppunit-header.h"


template<class T, T Polynomial>
class Test_Crc : public CppUnit::TestFixture
{
	using calc_type = Crc<T, Polynomial>;

	using crc_type = T;

	CPPUNIT_TEST_SUITE(Test_Crc);
	CPPUNIT_TEST(testThreadedCreation);
	CPPUNIT_TEST(crcCalculation);
	CPPUNIT_TEST(crcFile<uint8_t>);
	CPPUNIT_TEST(crcFile<uint16_t>);
	CPPUNIT_TEST(crcFile<uint64_t>);
	CPPUNIT_TEST(crcFile<uint64_t>);
	CPPUNIT_TEST_SUITE_END();

 public:
	~Test_Crc()
	{
		unlink("data");
	}

	void setUp()
	{
		const size_t filesize = (1 << 24);
		uint64_t x;
		struct stat st;
		int rc, size = 0;;

		// force deterministic sequence
		std::mt19937_64 generator(0);
		std::uniform_int_distribution<uint64_t> dist(
			std::numeric_limits<uint64_t>::min(),
			std::numeric_limits<uint64_t>::max());

		if ( stat("data", &st) != 0 )
		{
			if ((randomData = open("data", O_CREAT|O_RDWR| O_TRUNC, 0644)) < 0)
				throw std::runtime_error("Fail");

			for (size_t i = 0; i < filesize/sizeof(x); ++i)
			{
				x = dist(generator);
				rc = write(randomData, reinterpret_cast<char*>(&x), sizeof(x));

				if (rc != sizeof(x))
					throw std::runtime_error("Write failed");

				size+=rc;
			}

			// printf("data created with size %d\n", size);
		} else
			if ((randomData = open("data", O_RDWR )) < 0)
				throw std::runtime_error("Fail");

		lseek(randomData, 0, SEEK_SET);
	}

	void tearDown() {
		close(randomData);
		unlink("data");
	}

	void testThreadedCreation()
	{
		std::vector<std::thread> threads;
		for (int i = 0; i < 100; ++i)
		{
			threads.emplace_back([] {
				Crc<T, Polynomial> c;
			});
		}

		for (auto & i : threads) i.join();
	}

	void crcCalculation()
	{
		calc_type crc1;
		crc1(datum1.c_str(), datum1.size());
		CPPUNIT_ASSERT(crc1.get() == datum1crc);

		calc_type crc2;
		crc2(datum2.c_str(), datum2.size());
		CPPUNIT_ASSERT(crc2.get() == datum2crc);
{'fraction_non_alphanumeric': 0.16888994759409243, 'fraction_numerical': 0.024773701762744165, 'mean_word_length': 4.390243902439025, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2717761', 'n_tokens_mistral': 1946, 'n_tokens_neox': 1706, 'n_words': 327}