<commit_before>package fr.openwide.core.test.jpa.more.util.search.analysis.fr;

import junit.framework.Assert;

import org.junit.Test;

import fr.openwide.core.jpa.more.util.search.analysis.fr.CoreFrenchMinimalStemmer;

public class TestCoreFrenchMinimalStemmer {
	
	private static final CoreFrenchMinimalStemmer STEMMER = new CoreFrenchMinimalStemmer();
	
	@Test
	public void testStem() {
		Assert.assertEquals("stag", stem("stage"));
		Assert.assertEquals("stag", stem("stages"));
		
		Assert.assertEquals("cree", stem("creer"));
		
		Assert.assertEquals("noir", stem("noir"));
		Assert.assertEquals("noir", stem("noire"));
		Assert.assertEquals("noir", stem("noires"));
		Assert.assertEquals("noir", stem("noirs"));
		
		Assert.assertEquals("noir", stem("noir"));
		
		Assert.assertEquals("cheval", stem("cheval"));
		Assert.assertEquals("cheval", stem("chevaux"));
		
		Assert.assertEquals("chapeau", stem("chapeau"));
		Assert.assertEquals("chapeau", stem("chapeaux"));
	}
	
	public String stem(String word) {
		if (word == null) {
			throw new IllegalArgumentException("word cannot be null");
		}
		char[] charArray = word.toCharArray();
		int len = STEMMER.stem(charArray, word.length());
		return String.valueOf(charArray).substring(0, len);
	}

}<commit_msg>suppression d'un test en double<commit_after>package fr.openwide.core.test.jpa.more.util.search.analysis.fr;

import junit.framework.Assert;

import org.junit.Test;

import fr.openwide.core.jpa.more.util.search.analysis.fr.CoreFrenchMinimalStemmer;

public class TestCoreFrenchMinimalStemmer {
	
	private static final CoreFrenchMinimalStemmer STEMMER = new CoreFrenchMinimalStemmer();
	
	@Test
	public void testStem() {
		Assert.assertEquals("stag", stem("stage"));
		Assert.assertEquals("stag", stem("stages"));
		
		Assert.assertEquals("cree", stem("creer"));
		
		Assert.assertEquals("noir", stem("noir"));
		Assert.assertEquals("noir", stem("noire"));
		Assert.assertEquals("noir", stem("noires"));
		Assert.assertEquals("noir", stem("noirs"));
		
		Assert.assertEquals("cheval", stem("cheval"));
		Assert.assertEquals("cheval", stem("chevaux"));
		
		Assert.assertEquals("chapeau", stem("chapeau"));
		Assert.assertEquals("chapeau", stem("chapeaux"));
	}
	
	public String stem(String word) {
		if (word == null) {
			throw new IllegalArgumentException("word cannot be null");
		}
		char[] charArray = word.toCharArray();
		int len = STEMMER.stem(charArray, word.length());
		return String.valueOf(charArray).substring(0, len);
	}

}
{'fraction_non_alphanumeric': 0.1645367412140575, 'fraction_numerical': 0.0007987220447284345, 'mean_word_length': 5.7158176943699734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5717174', 'n_tokens_mistral': 868, 'n_tokens_neox': 774, 'n_words': 142}