<commit_before>package outbackcdx;

import org.junit.Test;

import static org.junit.Assert.*;

public class FilterTest {
    @Test
    public void testRegexFilter() {
        Capture one = new Capture();
        one.file = "one.warc.gz";
        one.status = 201;
        Capture two = new Capture();
        two.file = "two.warc.gz";
        two.status = 202;

        assertTrue(Filter.fromSpec("filename:one.*").test(one));
        assertFalse(Filter.fromSpec("filename:one.*").test(two));
        assertTrue(Filter.fromSpec("status:20.").test(one));
        assertTrue(Filter.fromSpec("status:20.").test(two));
        assertTrue(Filter.fromSpec("status:201").test(one));
        assertFalse(Filter.fromSpec("status:201").test(two));
        assertTrue(Filter.fromSpec("!status:201").test(two));
    }

    @Test
    public void testSubstringFilter() {
        Capture one = new Capture();
        one.file = "one.warc.gz";
        one.status = 201;
        Capture two = new Capture();
        two.file = "two.warc.gz";
        two.status = 202;

        assertTrue(Filter.fromSpec("~filename:one").test(one));
        assertFalse(Filter.fromSpec("~filename:one").test(two));
        assertTrue(Filter.fromSpec("~status:20").test(one));
        assertTrue(Filter.fromSpec("~status:20").test(two));
        assertTrue(Filter.fromSpec("~status:201").test(one));
        assertFalse(Filter.fromSpec("~status:201").test(two));
        assertTrue(Filter.fromSpec("~!status:201").test(two));
    }

    @Test
    public void testCollapse() {
        Capture one = new Capture();
        one.original = "http://example.com/collapse";
        one.timestamp = 20190101000000l;
        Capture two = new Capture();
        two.original = "http://example.com/collapse";
        two.timestamp = 20190101000005l;

        Filter originalCollapser = Filter.collapser("original");
        assertTrue(originalCollapser.test(one));
        assertFalse(originalCollapser.test(two));

        Filter t14Collapser = Filter.collapser("timestamp");
        assertTrue(t14Collapser.test(one));
        assertTrue(t14Collapser.test(two));

        Filter t13Collapser = Filter.collapser("timestamp:13");
        assertTrue(t13Collapser.test(one));
        assertFalse(t13Collapser.test(two));

        Filter t11Collapser = Filter.collapser("timestamp:11");
        assertTrue(t11Collapser.test(one));
        assertFalse(t11Collapser.test(two));
    }
<commit_msg>remove accidentally committed pointless test<commit_after>package outbackcdx;

import org.junit.Test;

import static org.junit.Assert.*;

public class FilterTest {
    @Test
    public void testRegexFilter() {
        Capture one = new Capture();
        one.file = "one.warc.gz";
        one.status = 201;
        Capture two = new Capture();
        two.file = "two.warc.gz";
        two.status = 202;

        assertTrue(Filter.fromSpec("filename:one.*").test(one));
        assertFalse(Filter.fromSpec("filename:one.*").test(two));
        assertTrue(Filter.fromSpec("status:20.").test(one));
        assertTrue(Filter.fromSpec("status:20.").test(two));
        assertTrue(Filter.fromSpec("status:201").test(one));
        assertFalse(Filter.fromSpec("status:201").test(two));
        assertTrue(Filter.fromSpec("!status:201").test(two));
    }

    @Test
    public void testSubstringFilter() {
        Capture one = new Capture();
        one.file = "one.warc.gz";
        one.status = 201;
        Capture two = new Capture();
        two.file = "two.warc.gz";
        two.status = 202;

        assertTrue(Filter.fromSpec("~filename:one").test(one));
        assertFalse(Filter.fromSpec("~filename:one").test(two));
        assertTrue(Filter.fromSpec("~status:20").test(one));
        assertTrue(Filter.fromSpec("~status:20").test(two));
        assertTrue(Filter.fromSpec("~status:201").test(one));
        assertFalse(Filter.fromSpec("~status:201").test(two));
        assertTrue(Filter.fromSpec("~!status:201").test(two));
    }

    @Test
    public void testCollapse() {
        Capture one = new Capture();
        one.original = "http://example.com/collapse";
        one.timestamp = 20190101000000l;
        Capture two = new Capture();
        two.original = "http://example.com/collapse";
        two.timestamp = 20190101000005l;

        Filter originalCollapser = Filter.collapser("original");
        assertTrue(originalCollapser.test(one));
        assertFalse(originalCollapser.test(two));

        Filter t14Collapser = Filter.collapser("timestamp");
        assertTrue(t14Collapser.test(one));
        assertTrue(t14Collapser.test(two));

        Filter t13Collapser = Filter.collapser("timestamp:13");
        assertTrue(t13Collapser.test(one));
        assertFalse(t13Collapser.test(two));
    }

{'fraction_non_alphanumeric': 0.1547494233591948, 'fraction_numerical': 0.03522751100859719, 'mean_word_length': 3.0117746005046255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3807605', 'n_tokens_mistral': 1553, 'n_tokens_neox': 1429, 'n_words': 207}