<commit_before>package org.elasticsearch.index.analysis;

import com.huaban.analysis.jieba.JiebaSegmenter;
import com.huaban.analysis.jieba.WordDictionary;
import org.apache.lucene.analysis.Tokenizer;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;

public class JiebaTokenizerFactory extends AbstractTokenizerFactory {

  private String segMode;

  public JiebaTokenizerFactory(IndexSettings indexSettings, String name, Settings settings) {
    super(indexSettings, name, settings);
    this.segMode = settings.get("segMode", JiebaSegmenter.SegMode.SEARCH.name());
    Environment env = new Environment(indexSettings.getSettings());
    WordDictionary.getInstance().init(env.pluginsFile().resolve("jieba/dic"));
  }

  @Override
  public Tokenizer create() {
    return new JiebaTokenizer(segMode);
  }

  public String getSegMode() {
    return segMode;
  }

  public void setSegMode(String segMode) {
    this.segMode = segMode;
  }

  public static TokenizerFactory getJiebaSearchTokenizerFactory(IndexSettings indexSettings,
                                                                Environment environment,
                                                                String s,
                                                                Settings settings) {
    JiebaTokenizerFactory jiebaTokenizerFactory = new JiebaTokenizerFactory(indexSettings, s,
        settings);
    jiebaTokenizerFactory.setSegMode(JiebaSegmenter.SegMode.SEARCH.name());
    return jiebaTokenizerFactory;
  }

  public static TokenizerFactory getJiebaIndexTokenizerFactory(IndexSettings indexSettings,
                                                               Environment environment,
                                                               String s,
                                                               Settings settings) {
    JiebaTokenizerFactory jiebaTokenizerFactory = new JiebaTokenizerFactory(indexSettings, s,
        settings);
    jiebaTokenizerFactory.setSegMode(JiebaSegmenter.SegMode.INDEX.name());
    return jiebaTokenizerFactory;
  }
}<commit_msg>get settings from elasticsearch<commit_after>package org.elasticsearch.index.analysis;

import com.huaban.analysis.jieba.JiebaSegmenter;
import com.huaban.analysis.jieba.WordDictionary;
import org.apache.logging.log4j.core.appender.SyslogAppender;
import org.apache.lucene.analysis.Tokenizer;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;

public class JiebaTokenizerFactory extends AbstractTokenizerFactory {

  private String segMode;


  public JiebaTokenizerFactory(IndexSettings indexSettings, Environment env,
                               String name, Settings settings) {
    super(indexSettings, name, settings);
    System.out.println(env.pluginsFile());
    WordDictionary.getInstance().init(env.pluginsFile().resolve("jieba/dic"));
  }

  @Override
  public Tokenizer create() {
    return new JiebaTokenizer(segMode);
  }

  public String getSegMode() {
    return segMode;
  }

  public void setSegMode(String segMode) {
    this.segMode = segMode;
  }

  public static TokenizerFactory getJiebaSearchTokenizerFactory(IndexSettings indexSettings,
                                                                Environment environment,
                                                                String s,
                                                                Settings settings) {
    JiebaTokenizerFactory jiebaTokenizerFactory = new JiebaTokenizerFactory(indexSettings,
        environment,
        s,
        settings);
    jiebaTokenizerFactory.setSegMode(JiebaSegmenter.SegMode.SEARCH.name());
    return jiebaTokenizerFactory;
  }

  public static TokenizerFactory getJiebaIndexTokenizerFactory(IndexSettings indexSettings,
                                                               Environment environment,
                                                               String s,
                                                               Settings settings) {
    JiebaTokenizerFactory jiebaTokenizerFactory = new JiebaTokenizerFactory(indexSettings,
        environment,
        s,
        settings);
    jiebaTokenizerFactory.setSegMode(JiebaSegmenter.SegMode.INDEX.name());
    return jiebaTokenizerFactory;
  }
}
{'fraction_non_alphanumeric': 0.06800723000451875, 'fraction_numerical': 0.00022593764121102577, 'mean_word_length': 2.257542310522443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4334866', 'n_tokens_mistral': 1051, 'n_tokens_neox': 1040, 'n_words': 199}