<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.commons.codec.language.bm;

import static org.junit.Assert.assertTrue;

import java.util.Arrays;
import java.util.List;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

/**
 * Tests guessLanguages API.
 * 
 * @author Apache Software Foundation
 * @since 2.0
 */
@RunWith(Parameterized.class)
public class LanguageGuessingTest {

    private static String EXACT = "exact";
    private static String ONE_OF = "one of";

    @Parameterized.Parameters
    public static List<Object[]> data() {
        return Arrays.asList(new Object[][] {
                { "Renault", "french", EXACT },
                { "Mickiewicz", "polish", EXACT },
                { "Thompson", "english", ONE_OF }, // this also hits german and greeklatin
                { "Nuñez", "spanish", EXACT },
                { "Carvalho", "portuguese", EXACT },
                { "Čapek", "czech", EXACT },
                { "Sjneijder", "dutch", EXACT },
                { "Klausewitz", "german", EXACT },
                { "Küçük", "turkish", EXACT },
                { "Giacometti", "italian", EXACT },
                { "Nagy", "hungarian", EXACT },
                { "Ceauşescu", "romanian", EXACT },
                { "Angelopoulos", "greeklatin", EXACT },
                { "Αγγελόπουλος", "greek", EXACT },
                { "Пушкин", "cyrillic", EXACT },
                { "כהן", "hebrew", EXACT },
                { "ácz", "any", EXACT },
                { "átz", "any", EXACT } });
    }

    private final String exactness;

    private final Lang lang = Lang.instance(NameType.GENERIC);
    private final String language;
    private final String name;

    public LanguageGuessingTest(String name, String language, String exactness) {
        this.name = name;
        this.language = language;
        this.exactness = exactness;
    }

    @Test
    public void testLanguageGuessing() {
        Languages.LanguageSet guesses = this.lang.guessLanguages(this.name);

        assertTrue("language predicted for name '" + this.name + "' is wrong: " + guesses + " should contain '" + this.language + "'",
                guesses.contains(this.language));

    }
}
<commit_msg>CODEC-127 Convert to use Unicode except in comments<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.commons.codec.language.bm;

import static org.junit.Assert.assertTrue;

import java.util.Arrays;
import java.util.List;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

/**
 * Tests guessLanguages API.
 * 
 * @author Apache Software Foundation
 * @since 2.0
 */
@RunWith(Parameterized.class)
public class LanguageGuessingTest {

    private static String EXACT = "exact";
    private static String ONE_OF = "one of";

    @Parameterized.Parameters
    public static List<Object[]> data() {
        return Arrays.asList(new Object[][] {
                { "Renault", "french", EXACT },
                { "Mickiewicz", "polish", EXACT },
                { "Thompson", "english", ONE_OF }, // this also hits german and greeklatin
                { "Nu\u00f1ez", "spanish", EXACT }, // Nuñez
                { "Carvalho", "portuguese", EXACT },
                { "\u010capek", "czech", EXACT }, // Čapek
                { "Sjneijder", "dutch", EXACT },
                { "Klausewitz", "german", EXACT },
                { "K\u00fc\u00e7\u00fck", "turkish", EXACT }, // Küçük
                { "Giacometti", "italian", EXACT },
                { "Nagy", "hungarian", EXACT },
                { "Ceau\u015fescu", "romanian", EXACT }, // Ceauşescu
                { "Angelopoulos", "greeklatin", EXACT },
                { "\u0391\u03b3\u03b3\u03b5\u03bb\u03cc\u03c0\u03bf\u03c5\u03bb\u03bf\u03c2", "greek", EXACT }, // Αγγελόπουλος
                { "\u041f\u0443\u0448\u043a\u0438\u043d", "cyrillic", EXACT }, // Пушкин
                { "\u05db\u05d4\u05df", "hebrew", EXACT }, // כהן
                { "\u00e1cz", "any", EXACT }, // ácz
                { "\u00e1tz", "any", EXACT } }); // átz
    }

    private final String exactness;

    private final Lang lang = Lang.instance(NameType.GENERIC);
    private final String language;
    private final String name;

    public LanguageGuessingTest(String name, String language, String exactness) {
        this.name = name;
        this.language = language;
        this.exactness = exactness;
    }

    @Test
    public void testLanguageGuessing() {
        Languages.LanguageSet guesses = this.lang.guessLanguages(this.name);

        assertTrue("language predicted for name '" + this.name + "' is wrong: " + guesses + " should contain '" + this.language + "'",
                guesses.contains(this.language));

    }
}

{'fraction_non_alphanumeric': 0.11849984242042232, 'fraction_numerical': 0.015285219035612984, 'mean_word_length': 2.526111111111111, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6649054', 'n_tokens_mistral': 1975, 'n_tokens_neox': 1844, 'n_words': 554}