<commit_before>/*
 * Copyright (C) 2012 <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
package com.github.fhirschmann.clozegen.lib.parser;

import com.github.fhirschmann.clozegen.lib.multiset.MapMultiset;
import com.github.fhirschmann.clozegen.lib.frequency.FrequencyParser;
import com.google.common.collect.Multiset;
import com.google.common.io.Resources;
import java.io.IOException;
import java.net.URL;
import junit.framework.TestCase;
import org.junit.Before;
import org.junit.Test;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class FrequencyParserTest extends TestCase {
    private URL trigrams;
    private URL bigrams;

    @Before
    public void setUp() {
        trigrams = Resources.getResource(
                "com/github/fhirschmann/clozegen/lib/frequency/trigrams.txt");
        bigrams = Resources.getResource(
                "com/github/fhirschmann/clozegen/lib/frequency/bigrams.txt");
    }

    @Test
    public void testParseMultiset() throws IOException {
        Multiset<String> ms = FrequencyParser.parseMultiset(trigrams);
        assertEquals(806, ms.count("one of the"));
        assertEquals(188, ms.count("and in the"));
        assertEquals(174, ms.count("because of the"));
    }

    public void testParseMapMultiset() throws IOException {
        MapMultiset<String, String> mms = FrequencyParser.parseMapMultiset(bigrams, 0);
        assertEquals(1404, mms.get("one").count("of"));

        MapMultiset<String, String> mms2 = FrequencyParser.parseMapMultiset(bigrams, 1);
        assertEquals(436, mms2.get("of").count("because"));
    }
}
<commit_msg>added priv const. test for freq parser<commit_after>/*
 * Copyright (C) 2012 <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 */
package com.github.fhirschmann.clozegen.lib.parser;

import com.github.fhirschmann.clozegen.lib.multiset.MapMultiset;
import com.github.fhirschmann.clozegen.lib.frequency.FrequencyParser;
import com.google.common.collect.Multiset;
import com.google.common.io.Resources;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URL;
import junit.framework.TestCase;
import org.junit.Before;
import org.junit.Test;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class FrequencyParserTest extends TestCase {
    private URL trigrams;
    private URL bigrams;

    @Before
    public void setUp() {
        trigrams = Resources.getResource(
                "com/github/fhirschmann/clozegen/lib/frequency/trigrams.txt");
        bigrams = Resources.getResource(
                "com/github/fhirschmann/clozegen/lib/frequency/bigrams.txt");
    }

    @Test
    public void testParseMultiset() throws IOException {
        Multiset<String> ms = FrequencyParser.parseMultiset(trigrams);
        assertEquals(806, ms.count("one of the"));
        assertEquals(188, ms.count("and in the"));
        assertEquals(174, ms.count("because of the"));
    }

    public void testParseMapMultiset() throws IOException {
        MapMultiset<String, String> mms = FrequencyParser.parseMapMultiset(bigrams, 0);
        assertEquals(1404, mms.get("one").count("of"));

        MapMultiset<String, String> mms2 = FrequencyParser.parseMapMultiset(bigrams, 1);
        assertEquals(436, mms2.get("of").count("because"));
    }

    /**
     * Hack to exclude the private constructor from code coverage metrics.
     */
    @Test
    public void testPrivateConstructor() throws Exception {
        Constructor<?>[] cons = FrequencyParser.class.getDeclaredConstructors();
        cons[0].setAccessible(true);
        cons[0].newInstance((Object[]) null);
    }
}

{'fraction_non_alphanumeric': 0.10437235543018336, 'fraction_numerical': 0.01611928269192021, 'mean_word_length': 3.6090993500464252, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3144822', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1380, 'n_words': 436}