<commit_before>package cscie99.team2.lingolearn.server.confuser;

import static org.junit.Assert.assertEquals;

import java.io.IOException;

import org.junit.Test;

import cscie99.team2.lingolearn.shared.error.ConfuserException;

public class ConfuserToolsTest {
	/**
	 * Check to make sure the black list is running correctly by using
	 * a phrase from the middle of the list. 
	 */
	@Test
	public void testOnBlackList() throws ConfuserException, IOException {
		// Check the phrase "Stop being stupid" against the blacklist
		assertEquals(true, ConfuserTools.onBlackList("ふざけるな", "jp"));
	}
	
	/**
	 * Check the detection of Unicode ranges by using the standard ranges from
	 * http://www.unicode.org/charts/
	 */
	@Test
	public void testCheckCharacter() throws ConfuserException {
		// Check to make sure unknowns work by checking against basic ASCII
		for (char ndx = 0x021; ndx < 0x007E; ndx++) {
			assertEquals(CharacterType.Unknown, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
		// Check to make sure Hiragana pass
		for (char ndx = 0x3041; ndx < 0x3096; ndx++) {
			assertEquals(CharacterType.Hiragana, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
		// Check to make sure the katakana pass
		for (char ndx = 0x30A1; ndx < 0x30FC; ndx++) {
			assertEquals(CharacterType.Katakana, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
		// Check to make sure the kanji pass, note that we are checking the 
		// full CJK range so this includes kanji that are not used in Japanese
		for (char ndx = 0x4E00; ndx < 0x9FAF; ndx++) {
			assertEquals(CharacterType.Kanji, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
	}
}<commit_msg>#3 Added a false check for the onBlacklist() unit test.<commit_after>package cscie99.team2.lingolearn.server.confuser;

import static org.junit.Assert.assertEquals;

import java.io.IOException;

import org.junit.Test;

import cscie99.team2.lingolearn.shared.error.ConfuserException;

public class ConfuserToolsTest {
	/**
	 * Check to make sure the black list is running correctly by using
	 * a phrase from the middle of the list. 
	 */
	@Test
	public void testOnBlackList() throws ConfuserException, IOException {
		// Check the phrase "Stop being stupid" against the blacklist
		assertEquals(true, ConfuserTools.onBlackList("ふざけるな", "jp"));
		// Check to phrase "Hello" against the blacklist
		assertEquals(false, ConfuserTools.onBlackList("こんにちは", "jp"));
	}
	
	/**
	 * Check the detection of Unicode ranges by using the standard ranges from
	 * http://www.unicode.org/charts/
	 */
	@Test
	public void testCheckCharacter() throws ConfuserException {
		// Check to make sure unknowns work by checking against basic ASCII
		for (char ndx = 0x021; ndx < 0x007E; ndx++) {
			assertEquals(CharacterType.Unknown, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
		// Check to make sure Hiragana pass
		for (char ndx = 0x3041; ndx < 0x3096; ndx++) {
			assertEquals(CharacterType.Hiragana, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
		// Check to make sure the katakana pass
		for (char ndx = 0x30A1; ndx < 0x30FC; ndx++) {
			assertEquals(CharacterType.Katakana, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
		// Check to make sure the kanji pass, note that we are checking the 
		// full CJK range so this includes kanji that are not used in Japanese
		for (char ndx = 0x4E00; ndx < 0x9FAF; ndx++) {
			assertEquals(CharacterType.Kanji, ConfuserTools.checkCharacter(String.valueOf(ndx)));
		}
	}
}
{'fraction_non_alphanumeric': 0.10297482837528604, 'fraction_numerical': 0.02145308924485126, 'mean_word_length': 4.290468986384266, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7060300', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1088, 'n_words': 335}