<commit_before>package org.codelibs.elasticsearch.ja.analysis;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.codelibs.analysis.ja.PosConcatenationFilter;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
import org.elasticsearch.index.settings.IndexSettingsService;

public class PosConcatenationFilterFactory extends AbstractTokenFilterFactory {

    private Set<String> posTags;

    @Inject
    public PosConcatenationFilterFactory(Index index, IndexSettingsService indexSettingsService, @Assisted String name,
            @Assisted Settings settings, Environment env) {
        super(index, indexSettingsService.getSettings(), name, settings);

        String posPath = settings.get("tags_path");

        if (posPath != null) {
            File posFile = env.configFile().resolve(posPath).toFile();
            try (InputStream is = new FileInputStream(posFile)) {
                posTags.addAll(WordlistLoader.getLines(is, StandardCharsets.UTF_8));
            } catch (IOException e) {
                throw new IllegalArgumentException("Could not load " + posFile.getAbsolutePath(), e);
            }
        } else {
            String tags = settings.get("tags");
            if (tags != null) {
                for (String tag : tags.split(",")) {
                    posTags.add(tag.trim());
                }
            } else {
                posTags = new HashSet<>();
            }
        }
    }

    @Override
    public TokenStream create(TokenStream tokenStream) {
        return new PosConcatenationFilter(tokenStream, posTags);
    }
}<commit_msg>modify parameter handling<commit_after>package org.codelibs.elasticsearch.ja.analysis;

import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.TokenStream;
import org.codelibs.analysis.ja.PosConcatenationFilter;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.assistedinject.Assisted;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
import org.elasticsearch.index.analysis.Analysis;
import org.elasticsearch.index.settings.IndexSettingsService;

public class PosConcatenationFilterFactory extends AbstractTokenFilterFactory {

    private Set<String> posTags = new HashSet<>();

    @Inject
    public PosConcatenationFilterFactory(Index index, IndexSettingsService indexSettingsService, @Assisted String name,
            @Assisted Settings settings, Environment env) {
        super(index, indexSettingsService.getSettings(), name, settings);

        List<String> tagList = Analysis.getWordList(env, settings, "tags");
        if (tagList != null) {
            posTags.addAll(tagList);
        }
    }

    @Override
    public TokenStream create(TokenStream tokenStream) {
        return new PosConcatenationFilter(tokenStream, posTags);
    }
}
{'fraction_non_alphanumeric': 0.09598853868194843, 'fraction_numerical': 0.00028653295128939826, 'mean_word_length': 3.692204301075269, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6350347', 'n_tokens_mistral': 931, 'n_tokens_neox': 930, 'n_words': 189}