<commit_before>package org.apache.hadoop.util.bloom;

import org.apache.hadoop.util.hash.Hash;
import org.junit.Test;

import java.nio.charset.Charset;
import java.util.*;

import static org.junit.Assert.*;

public class CountingBloomFilterTest {
    private static final Charset UTF8 = Charset.forName("UTF-8");

    @Test
    public void testBasics() {
        final CountingBloomFilter cbf = new CountingBloomFilter(100, 2, Hash.MURMUR_HASH);
        final Key key = new Key("abc".getBytes(UTF8));
        cbf.add(key);
        assertTrue("key present after insertion", cbf.membershipTest(key));

        cbf.delete(key);
        assertFalse("key gone after deletion", cbf.membershipTest(key));
    }


    private static final Random RANDOM = new Random();
    private static final int LARGE_VECTOR_SIZE = 2 << 29;
    private static final int NBHASH = 5;

    @Test
    public void testDoesNotUseMoreThanOneGigabyteOfMemory() {
        assertTrue(LARGE_VECTOR_SIZE > 0);

        final int numberOfQwords = CountingBloomFilter.buckets2words(LARGE_VECTOR_SIZE);
        final int bytesPerQword = 8;
        final int bytesForCBF = bytesPerQword * numberOfQwords;

        System.out.println("bytes = " + bytesForCBF);
        System.out.println("items = " + LARGE_VECTOR_SIZE);
        assertTrue(bytesForCBF < /* 1 GiB */ (1048576L * 1024));
    }

    @Test
    public void testManyItems() {<commit_msg>Verified memory use expectations<commit_after>package org.apache.hadoop.util.bloom;

import org.apache.hadoop.util.hash.Hash;
import org.junit.Test;

import java.nio.charset.Charset;
import java.util.*;

import static org.junit.Assert.*;

public class CountingBloomFilterTest {
    private static final Charset UTF8 = Charset.forName("UTF-8");

    @Test
    public void testBasics() {
        final CountingBloomFilter cbf = new CountingBloomFilter(100, 2, Hash.MURMUR_HASH);
        final Key key = new Key("abc".getBytes(UTF8));
        cbf.add(key);
        assertTrue("key present after insertion", cbf.membershipTest(key));

        cbf.delete(key);
        assertFalse("key gone after deletion", cbf.membershipTest(key));
    }


    private static final Random RANDOM = new Random();
    private static final int LARGE_VECTOR_SIZE = 2 << 29;
    private static final int NBHASH = 5;

    @Test
    public void testDoesNotUseMoreThanOneGigabyteOfMemory() {
        assertTrue(LARGE_VECTOR_SIZE > 0);

        final int expectedMemorySize = LARGE_VECTOR_SIZE / 8 /* bits per byte */ * 4 /* bits per counter */;
        final int bytesPerQword = 8;
        final int numberOfQwords = CountingBloomFilter.buckets2words(LARGE_VECTOR_SIZE);
        final int bytesForCBF = bytesPerQword * numberOfQwords;

        System.out.println("bytes = " + bytesForCBF);
        System.out.println("items = " + LARGE_VECTOR_SIZE);
        assertEquals(expectedMemorySize, bytesForCBF);
        assertTrue(bytesForCBF < /* 1 GiB */ (1048576L * 1024));
    }

    @Test
    public void testManyItems() {
{'fraction_non_alphanumeric': 0.10396534488503832, 'fraction_numerical': 0.017994001999333556, 'mean_word_length': 3.2885714285714287, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3437886', 'n_tokens_mistral': 993, 'n_tokens_neox': 945, 'n_words': 221}