<commit_before>package wordcram.text;

import java.util.*;

import org.junit.Assert;
import org.junit.Test;

import wordcram.Word;

/*
 Copyright 2010 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

public class TestWordCounter {
	
	private Comparator<Word> alphabetically = new Comparator<Word>() {
		public int compare(Word word1, Word word2) {
			return word1.word.compareTo(word2.word);
		}
	};

	@Test
	public void testPunctuationInStopWords() {
		WordCounter wc = new WordCounter("don't i'll");
		String[] words = "i don't want any more, can't you see i'll be ill?".split(" ");
		Word[] weightedWords = wc.count(words);		

		// Sort them by word, since they're all the same weight.
		Arrays.sort(weightedWords, alphabetically);

		String[] expectedWords = "any be can't i ill? more, see want you"
				.split(" ");

		Assert.assertEquals(expectedWords.length, weightedWords.length);
		for (int i = 0; i < weightedWords.length; i++) {
			Assert.assertEquals(expectedWords[i], weightedWords[i].word);
			Assert.assertEquals(1.0, weightedWords[i].weight, 0.0f);
		}
	}

	@Test
	public void testCountsWithStopWords() {
		WordCounter wc = new WordCounter("these are stop words");

		String[] words = "biscuit biscuit cocoa cherry cherry cherry stop words are these these are stop words"
				.split(" ");
		Word[] weightedWords = wc.count(words);
		Arrays.sort(weightedWords);

		Assert.assertEquals(3, weightedWords.length);

		Assert.assertEquals("cherry", weightedWords[0].word);
		Assert.assertEquals("biscuit", weightedWords[1].word);
		Assert.assertEquals("cocoa", weightedWords[2].word);

		Assert.assertEquals(3, (int) weightedWords[0].weight);
		Assert.assertEquals(2, (int) weightedWords[1].weight);
		Assert.assertEquals(1, (int) weightedWords[2].weight);
	}
	
	@Test
	public void testThatStopWordsAreCaseInsensitive() {
		WordCounter wc = new WordCounter("STOP WORDS");
		
		String[] words = "these are stop words Stop Words STOP WORDS".split(" ");
		Word[] weightedWords = wc.count(words);
		Arrays.sort(weightedWords, alphabetically);
		
		Assert.assertEquals(2, weightedWords.length);
		Assert.assertEquals("are", weightedWords[0].word);
		Assert.assertEquals("these", weightedWords[1].word);
	}

	@Test
	public void testCountsWithNoStopWords() {
		WordCounter wc = new WordCounter("");

		String[] words = "a b c a b a".split(" ");
		Word[] weightedWords = wc.count(words);
		Arrays.sort(weightedWords);

		Assert.assertEquals(3, weightedWords.length);

		Assert.assertEquals("a", weightedWords[0].word);
		Assert.assertEquals("b", weightedWords[1].word);
		Assert.assertEquals("c", weightedWords[2].word);

		Assert.assertEquals(3, (int) weightedWords[0].weight);
		Assert.assertEquals(2, (int) weightedWords[1].weight);
		Assert.assertEquals(1, (int) weightedWords[2].weight);
	}
}<commit_msg>refactored TestWordCounter: pulled out a helper method to make the tests clearer<commit_after>package wordcram.text;

import java.util.*;

import org.junit.Assert;
import org.junit.Test;

import wordcram.Word;

/*
 Copyright 2010 <NAME>

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

public class TestWordCounter {
	
	private Comparator<Word> alphabetically = new Comparator<Word>() {
		public int compare(Word word1, Word word2) {
			return word1.word.compareTo(word2.word);
		}
	};

	@Test
	public void testPunctuationInStopWords() {
		WordCounter wc = new WordCounter("don't i'll");
		String[] words = split("i don't want any more, can't you see i'll be ill?");
		Word[] weightedWords = wc.count(words);		

		// Sort them by word, since they're all the same weight.
		Arrays.sort(weightedWords, alphabetically);

		assertWeightedWordsAre(weightedWords, "any 1", "be 1", "can't 1",
				"i 1", "ill? 1", "more, 1", "see 1", "want 1", "you 1");
	}

	@Test
	public void testCountsWithStopWords() {
		WordCounter wc = new WordCounter("these are stop words");

		String[] words = split("biscuit biscuit cocoa cherry cherry cherry stop words are these these are stop words");
		Word[] weightedWords = wc.count(words);
		Arrays.sort(weightedWords);
		
		assertWeightedWordsAre(weightedWords, "cherry 3", "biscuit 2", "cocoa 1");
	}
	
	@Test
	public void testThatStopWordsAreCaseInsensitive() {
		WordCounter wc = new WordCounter("STOP WORDS");
		
		String[] words = split("these are stop words Stop Words STOP WORDS");
		Word[] weightedWords = wc.count(words);
		Arrays.sort(weightedWords, alphabetically);
		
		assertWeightedWordsAre(weightedWords, "are 1", "these 1");
	}

	@Test
	public void testCountsWithNoStopWords() {
		WordCounter wc = new WordCounter("");

		String[] words = split("a b c a b a");
		Word[] weightedWords = wc.count(words);
		Arrays.sort(weightedWords);
		
		assertWeightedWordsAre(weightedWords, "a 3", "b 2", "c 1");
	}
	
	private void assertWeightedWordsAre(Word[] actualWords, String... expectedCases) {
		
		Assert.assertEquals("Got the wrong number of words", expectedCases.length, actualWords.length);
		
		for (int i = 0; i < expectedCases.length; i++) {
			String[] wordAndWeight = split(expectedCases[i].trim());
			Assert.assertEquals(wordAndWeight[0], actualWords[i].word);
			Assert.assertEquals(Double.parseDouble(wordAndWeight[1]), actualWords[i].weight, 0.0001);
		}
	}
	
	private String[] split(String words) {
		return words.split(" ");
	}
}
{'fraction_non_alphanumeric': 0.12390368362302663, 'fraction_numerical': 0.012278743422101739, 'mean_word_length': 4.691470054446461, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5437034', 'n_tokens_mistral': 2173, 'n_tokens_neox': 1852, 'n_words': 589}