<commit_before>// test base bloom filter
#include "catch.hpp"

#include "BaseBloomFilter.hpp"

// define main

TEST_CASE( "testing BaseBloomFilter", "[bae_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	REQUIRE(bf.get_bit_count() == 1024 * 1024 * 32 * 10);
	REQUIRE(bf.get_hash_count() == 2);
	CHECK(bf.get_inserted_count() == 0);
}<commit_msg>added a test for BaseBloomFitler<commit_after>// test base bloom filter
#include "catch.hpp"

#include "BaseBloomFilter.hpp"

// define main

TEST_CASE( "testing BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	REQUIRE(bf.get_bit_count() == 1024 * 1024 * 32 * 10);
	REQUIRE(bf.get_hash_count() == 2);
	CHECK(bf.get_inserted_count() == 0);
}

TEST_CASE( "testing add BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	size_t expected_size = 1024 * 1024 * 32 * 10;
	
	bf.add(1);
	REQUIRE(bf.get_bit_count() == expected_size);
	REQUIRE(bf.get_hash_count() == 2);
	REQUIRE(bf.get_inserted_count() == 1);
}

TEST_CASE( "testing contains BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	size_t expected_size = 1024 * 1024 * 32 * 10;
	
	bf.add(1);
	REQUIRE(bf.get_bit_count() == expected_size);
	REQUIRE(bf.contains(1));
	// should not expect this collision since the BF so far is empty (except 
	// for 1 bit)
	REQUIRE(!bf.contains(0) );
}

TEST_CASE( "testing populate BaseBloomFilter", "[base_bloom_filter]" ) {
	BaseBloomFilter bf(20);
	size_t expected_size = 1024 * 1024 * 32 * 10;
	unordered_set<kmer_t> kmers;
	for (int i = 0; i < 1000; i++)
		kmers.emplace(i);

	bf.populate(kmers);
	REQUIRE(bf.get_bit_count() == expected_size);
	for (const auto & item : kmers)
		REQUIRE(bf.contains(item));
	REQUIRE(bf.get_inserted_count() == kmers.size());
}
{'fraction_non_alphanumeric': 0.15789473684210525, 'fraction_numerical': 0.04919908466819222, 'mean_word_length': 5.202127659574468, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '220752', 'n_tokens_mistral': 766, 'n_tokens_neox': 682, 'n_words': 144}