<commit_before>/*
 * Licensed to ElasticSearch and <NAME> under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. ElasticSearch licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.analysis;

import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ar.ArabicStemFilter;
import org.apache.lucene.analysis.br.BrazilianStemFilter;
import org.apache.lucene.analysis.cz.CzechStemFilter;
import org.apache.lucene.analysis.de.GermanLightStemFilter;
import org.apache.lucene.analysis.de.GermanMinimalStemFilter;
import org.apache.lucene.analysis.el.GreekStemFilter;
import org.apache.lucene.analysis.en.EnglishMinimalStemFilter;
import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
import org.apache.lucene.analysis.en.KStemFilter;
import org.apache.lucene.analysis.es.SpanishLightStemFilter;
import org.apache.lucene.analysis.fi.FinnishLightStemFilter;
import org.apache.lucene.analysis.fr.FrenchLightStemFilter;
import org.apache.lucene.analysis.fr.FrenchMinimalStemFilter;
import org.apache.lucene.analysis.hi.HindiStemFilter;
import org.apache.lucene.analysis.hu.HungarianLightStemFilter;
import org.apache.lucene.analysis.id.IndonesianStemFilter;
import org.apache.lucene.analysis.it.ItalianLightStemFilter;
import org.apache.lucene.analysis.pt.PortugueseLightStemFilter;
import org.apache.lucene.analysis.pt.PortugueseMinimalStemFilter;
import org.apache.lucene.analysis.pt.PortugueseStemFilter;
import org.apache.lucene.analysis.ru.RussianLightStemFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.settings.IndexSettings;
import org.tartarus.snowball.ext.*;

/**
 */
public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {

    private String language;

    @Inject
    public StemmerTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) {
        super(index, indexSettings, name, settings);
        this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter")));
    }

    @Override
    public TokenStream create(TokenStream tokenStream) {
        if ("arabic".equalsIgnoreCase(language)) {
            return new ArabicStemFilter(tokenStream);
        } else if ("armenian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new ArmenianStemmer());
        } else if ("basque".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new BasqueStemmer());
        } else if ("brazilian".equalsIgnoreCase(language)) {
            return new BrazilianStemFilter(tokenStream);
        } else if ("catalan".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new CatalanStemmer());
        } else if ("czech".equalsIgnoreCase(language)) {
            return new CzechStemFilter(tokenStream);
        } else if ("danish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new DanishStemmer());
        } else if ("dutch".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new DutchStemmer());
        } else if ("english".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new EnglishStemmer());
        } else if ("finnish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new FinnishStemmer());
        } else if ("french".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new FrenchStemmer());
        } else if ("german".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new GermanStemmer());
        } else if ("german2".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new German2Stemmer());
        } else if ("hungarian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new HungarianStemmer());
        } else if ("italian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new ItalianStemmer());
        } else if ("kp".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new KpStemmer());
        } else if ("kstem".equalsIgnoreCase(language)) {
            return new KStemFilter(tokenStream);
        } else if ("lovins".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new LovinsStemmer());
        } else if ("norwegian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new NorwegianStemmer());
        } else if ("porter".equalsIgnoreCase(language)) {
            return new PorterStemFilter(tokenStream);
        } else if ("porter2".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new PorterStemmer());
        } else if ("portuguese".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new PortugueseStemmer());
        } else if ("romanian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new RomanianStemmer());
        } else if ("russian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new RussianStemmer());
        } else if ("spanish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new SpanishStemmer());
        } else if ("swedish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new SwedishStemmer());
        } else if ("turkish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new TurkishStemmer());
        } else if ("minimal_english".equalsIgnoreCase(language) || "minimalEnglish".equalsIgnoreCase(language)) {
            return new EnglishMinimalStemFilter(tokenStream);
        } else if ("possessive_english".equalsIgnoreCase(language) || "possessiveEnglish".equalsIgnoreCase(language)) {
            return new EnglishPossessiveFilter(tokenStream);
        } else if ("light_finish".equalsIgnoreCase(language) || "lightFinish".equalsIgnoreCase(language)) {
            return new FinnishLightStemFilter(tokenStream);
        } else if ("light_french".equalsIgnoreCase(language) || "lightFrench".equalsIgnoreCase(language)) {
            return new FrenchLightStemFilter(tokenStream);
        } else if ("minimal_french".equalsIgnoreCase(language) || "minimalFrench".equalsIgnoreCase(language)) {
            return new FrenchMinimalStemFilter(tokenStream);
        } else if ("light_german".equalsIgnoreCase(language) || "lightGerman".equalsIgnoreCase(language)) {
            return new GermanLightStemFilter(tokenStream);
        } else if ("minimal_german".equalsIgnoreCase(language) || "minimalGerman".equalsIgnoreCase(language)) {
            return new GermanMinimalStemFilter(tokenStream);
        } else if ("hindi".equalsIgnoreCase(language)) {
            return new HindiStemFilter(tokenStream);
        } else if ("light_hungarian".equalsIgnoreCase(language) || "lightHungarian".equalsIgnoreCase(language)) {
            return new HungarianLightStemFilter(tokenStream);
        } else if ("indonesian".equalsIgnoreCase(language)) {
            return new IndonesianStemFilter(tokenStream);
        } else if ("light_italian".equalsIgnoreCase(language) || "lightItalian".equalsIgnoreCase(language)) {
            return new ItalianLightStemFilter(tokenStream);
        } else if ("light_portuguese".equalsIgnoreCase(language) || "lightPortuguese".equalsIgnoreCase(language)) {
            return new PortugueseLightStemFilter(tokenStream);
        } else if ("minimal_portuguese".equalsIgnoreCase(language) || "minimalPortuguese".equalsIgnoreCase(language)) {
            return new PortugueseMinimalStemFilter(tokenStream);
        } else if ("portuguese".equalsIgnoreCase(language)) {
            return new PortugueseStemFilter(tokenStream);
        } else if ("light_russian".equalsIgnoreCase(language) || "lightRussian".equalsIgnoreCase(language)) {
            return new RussianLightStemFilter(tokenStream);
        } else if ("light_spanish".equalsIgnoreCase(language) || "lightSpanish".equalsIgnoreCase(language)) {
            return new SpanishLightStemFilter(tokenStream);
        } else if ("light_swedish".equalsIgnoreCase(language) || "lightSwedish".equalsIgnoreCase(language)) {
            return new SpanishLightStemFilter(tokenStream);
        } else if ("greek".equalsIgnoreCase(language)) {
            return new GreekStemFilter(tokenStream);
        }
        return new SnowballFilter(tokenStream, language);
    }

}<commit_msg>Corrected erroneous stem filter (loads the Spanish, not the Swedish light stemmer):  } else if ("light_swedish".equalsIgnoreCase(language) || "lightSwedish".equalsIgnoreCase(language)) {             return new SpanishLightStemFilter(tokenStream); } to "return new SwedishLightStemFilter(tokenStream);"<commit_after>/*
 * Licensed to ElasticSearch and <NAME> under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. ElasticSearch licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.index.analysis;

import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ar.ArabicStemFilter;
import org.apache.lucene.analysis.br.BrazilianStemFilter;
import org.apache.lucene.analysis.cz.CzechStemFilter;
import org.apache.lucene.analysis.de.GermanLightStemFilter;
import org.apache.lucene.analysis.de.GermanMinimalStemFilter;
import org.apache.lucene.analysis.el.GreekStemFilter;
import org.apache.lucene.analysis.en.EnglishMinimalStemFilter;
import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
import org.apache.lucene.analysis.en.KStemFilter;
import org.apache.lucene.analysis.es.SpanishLightStemFilter;
import org.apache.lucene.analysis.fi.FinnishLightStemFilter;
import org.apache.lucene.analysis.fr.FrenchLightStemFilter;
import org.apache.lucene.analysis.fr.FrenchMinimalStemFilter;
import org.apache.lucene.analysis.hi.HindiStemFilter;
import org.apache.lucene.analysis.hu.HungarianLightStemFilter;
import org.apache.lucene.analysis.id.IndonesianStemFilter;
import org.apache.lucene.analysis.it.ItalianLightStemFilter;
import org.apache.lucene.analysis.pt.PortugueseLightStemFilter;
import org.apache.lucene.analysis.pt.PortugueseMinimalStemFilter;
import org.apache.lucene.analysis.pt.PortugueseStemFilter;
import org.apache.lucene.analysis.ru.RussianLightStemFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.sv.SwedishLightStemFilter;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.settings.IndexSettings;
import org.tartarus.snowball.ext.*;

/**
 */
public class StemmerTokenFilterFactory extends AbstractTokenFilterFactory {

    private String language;

    @Inject
    public StemmerTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) {
        super(index, indexSettings, name, settings);
        this.language = Strings.capitalize(settings.get("language", settings.get("name", "porter")));
    }

    @Override
    public TokenStream create(TokenStream tokenStream) {
        if ("arabic".equalsIgnoreCase(language)) {
            return new ArabicStemFilter(tokenStream);
        } else if ("armenian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new ArmenianStemmer());
        } else if ("basque".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new BasqueStemmer());
        } else if ("brazilian".equalsIgnoreCase(language)) {
            return new BrazilianStemFilter(tokenStream);
        } else if ("catalan".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new CatalanStemmer());
        } else if ("czech".equalsIgnoreCase(language)) {
            return new CzechStemFilter(tokenStream);
        } else if ("danish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new DanishStemmer());
        } else if ("dutch".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new DutchStemmer());
        } else if ("english".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new EnglishStemmer());
        } else if ("finnish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new FinnishStemmer());
        } else if ("french".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new FrenchStemmer());
        } else if ("german".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new GermanStemmer());
        } else if ("german2".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new German2Stemmer());
        } else if ("hungarian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new HungarianStemmer());
        } else if ("italian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new ItalianStemmer());
        } else if ("kp".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new KpStemmer());
        } else if ("kstem".equalsIgnoreCase(language)) {
            return new KStemFilter(tokenStream);
        } else if ("lovins".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new LovinsStemmer());
        } else if ("norwegian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new NorwegianStemmer());
        } else if ("porter".equalsIgnoreCase(language)) {
            return new PorterStemFilter(tokenStream);
        } else if ("porter2".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new PorterStemmer());
        } else if ("portuguese".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new PortugueseStemmer());
        } else if ("romanian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new RomanianStemmer());
        } else if ("russian".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new RussianStemmer());
        } else if ("spanish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new SpanishStemmer());
        } else if ("swedish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new SwedishStemmer());
        } else if ("turkish".equalsIgnoreCase(language)) {
            return new SnowballFilter(tokenStream, new TurkishStemmer());
        } else if ("minimal_english".equalsIgnoreCase(language) || "minimalEnglish".equalsIgnoreCase(language)) {
            return new EnglishMinimalStemFilter(tokenStream);
        } else if ("possessive_english".equalsIgnoreCase(language) || "possessiveEnglish".equalsIgnoreCase(language)) {
            return new EnglishPossessiveFilter(tokenStream);
        } else if ("light_finish".equalsIgnoreCase(language) || "lightFinish".equalsIgnoreCase(language)) {
            return new FinnishLightStemFilter(tokenStream);
        } else if ("light_french".equalsIgnoreCase(language) || "lightFrench".equalsIgnoreCase(language)) {
            return new FrenchLightStemFilter(tokenStream);
        } else if ("minimal_french".equalsIgnoreCase(language) || "minimalFrench".equalsIgnoreCase(language)) {
            return new FrenchMinimalStemFilter(tokenStream);
        } else if ("light_german".equalsIgnoreCase(language) || "lightGerman".equalsIgnoreCase(language)) {
            return new GermanLightStemFilter(tokenStream);
        } else if ("minimal_german".equalsIgnoreCase(language) || "minimalGerman".equalsIgnoreCase(language)) {
            return new GermanMinimalStemFilter(tokenStream);
        } else if ("hindi".equalsIgnoreCase(language)) {
            return new HindiStemFilter(tokenStream);
        } else if ("light_hungarian".equalsIgnoreCase(language) || "lightHungarian".equalsIgnoreCase(language)) {
            return new HungarianLightStemFilter(tokenStream);
        } else if ("indonesian".equalsIgnoreCase(language)) {
            return new IndonesianStemFilter(tokenStream);
        } else if ("light_italian".equalsIgnoreCase(language) || "lightItalian".equalsIgnoreCase(language)) {
            return new ItalianLightStemFilter(tokenStream);
        } else if ("light_portuguese".equalsIgnoreCase(language) || "lightPortuguese".equalsIgnoreCase(language)) {
            return new PortugueseLightStemFilter(tokenStream);
        } else if ("minimal_portuguese".equalsIgnoreCase(language) || "minimalPortuguese".equalsIgnoreCase(language)) {
            return new PortugueseMinimalStemFilter(tokenStream);
        } else if ("portuguese".equalsIgnoreCase(language)) {
            return new PortugueseStemFilter(tokenStream);
        } else if ("light_russian".equalsIgnoreCase(language) || "lightRussian".equalsIgnoreCase(language)) {
            return new RussianLightStemFilter(tokenStream);
        } else if ("light_spanish".equalsIgnoreCase(language) || "lightSpanish".equalsIgnoreCase(language)) {
            return new SpanishLightStemFilter(tokenStream);
        } else if ("light_swedish".equalsIgnoreCase(language) || "lightSwedish".equalsIgnoreCase(language)) {
            return new SwedishLightStemFilter(tokenStream);
        } else if ("greek".equalsIgnoreCase(language)) {
            return new GreekStemFilter(tokenStream);
        }
        return new SnowballFilter(tokenStream, language);
    }

}
{'fraction_non_alphanumeric': 0.10363627012292342, 'fraction_numerical': 0.0007200534896878054, 'mean_word_length': 4.083398692810458, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5046751', 'n_tokens_mistral': 5243, 'n_tokens_neox': 5075, 'n_words': 1102}