<commit_before>package org.ansj.lucene5;

import java.io.BufferedReader;
import java.io.File;
import java.io.StringReader;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.ansj.lucene.util.AnsjTokenizer;
import org.ansj.splitWord.analysis.DicAnalysis;
import org.ansj.splitWord.analysis.IndexAnalysis;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.nlpcn.commons.lang.util.IOUtil;
import org.nlpcn.commons.lang.util.StringUtil;

public class AnsjAnalyzer extends Analyzer {

	/**
	 * dic equals user , query equals to
	 * 
	 * @author ansj
	 *
	 */
	public static enum TYPE {
		index, query, to, dic, user
	}

	/** 自定义停用词 */
	private Set<String> filter;
	/** 是否查询分词 */
	private TYPE type;

	/**
	 * @param filter
	 *            停用词
	 * @param pstemming
	 *            是否分析词干
	 */
	public AnsjAnalyzer(TYPE type, Set<String> filter) {
		this.type = type;
		this.filter = filter;
	}

	public AnsjAnalyzer(TYPE type, String stopwordsDir) {
		this.type = type;
		this.filter = filter(stopwordsDir);
	}

	public AnsjAnalyzer(TYPE type) {
		this.type = type;
	}

	public AnsjAnalyzer() {
	}

	private Set<String> filter(String stopwordsDir) {
		if (StringUtil.isBlank(stopwordsDir)) {
			return null;
		}
		try {
			List<String> readFile2List = IOUtil.readFile2List(stopwordsDir, IOUtil.UTF8);
			return new HashSet<String>(readFile2List);
		} catch (Exception e) {
			System.err.println("not foun stop word path by " + new File(stopwordsDir).getAbsolutePath());
			e.printStackTrace();
		}
		return null;
	}

	@Override
	protected TokenStreamComponents createComponents(String text) {
		BufferedReader reader = new BufferedReader(new StringReader(text));
		Tokenizer tokenizer = null;

		tokenizer = getTokenizer(reader, this.type, this.filter);
		return new TokenStreamComponents(tokenizer);
	}

	/**
	 * 获得一个tokenizer
	 * 
	 * @param reader
	 * @param type
	 * @param filter
	 * @return
	 */
	public static Tokenizer getTokenizer(BufferedReader reader, TYPE type, Set<String> filter) {
		Tokenizer tokenizer;

		switch (type) {
		case index:
			tokenizer = new AnsjTokenizer(new IndexAnalysis(reader), filter);
			break;
		case dic:
		case user:
			tokenizer = new AnsjTokenizer(new DicAnalysis(reader), filter);
			break;

		case to:
		case query:
			tokenizer = new AnsjTokenizer(new ToAnalysis(reader), filter);
			break;
		default:
			tokenizer = new AnsjTokenizer(new ToAnalysis(reader), filter);
		}

		return tokenizer;
	}
}<commit_msg>set toanalysis alias search<commit_after>package org.ansj.lucene5;

import java.io.BufferedReader;
import java.io.File;
import java.io.StringReader;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.ansj.lucene.util.AnsjTokenizer;
import org.ansj.splitWord.analysis.DicAnalysis;
import org.ansj.splitWord.analysis.IndexAnalysis;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.nlpcn.commons.lang.util.IOUtil;
import org.nlpcn.commons.lang.util.StringUtil;

public class AnsjAnalyzer extends Analyzer {

	/**
	 * dic equals user , query equals to
	 * 
	 * @author ansj
	 *
	 */
	public static enum TYPE {
		index, query, to, dic, user, search
	}

	/** 自定义停用词 */
	private Set<String> filter;
	/** 是否查询分词 */
	private TYPE type;

	/**
	 * @param filter
	 *            停用词
	 * @param pstemming
	 *            是否分析词干
	 */
	public AnsjAnalyzer(TYPE type, Set<String> filter) {
		this.type = type;
		this.filter = filter;
	}

	public AnsjAnalyzer(TYPE type, String stopwordsDir) {
		this.type = type;
		this.filter = filter(stopwordsDir);
	}

	public AnsjAnalyzer(TYPE type) {
		this.type = type;
	}

	public AnsjAnalyzer() {
	}

	private Set<String> filter(String stopwordsDir) {
		if (StringUtil.isBlank(stopwordsDir)) {
			return null;
		}
		try {
			List<String> readFile2List = IOUtil.readFile2List(stopwordsDir, IOUtil.UTF8);
			return new HashSet<String>(readFile2List);
		} catch (Exception e) {
			System.err.println("not foun stop word path by " + new File(stopwordsDir).getAbsolutePath());
			e.printStackTrace();
		}
		return null;
	}

	@Override
	protected TokenStreamComponents createComponents(String text) {
		BufferedReader reader = new BufferedReader(new StringReader(text));
		Tokenizer tokenizer = null;

		tokenizer = getTokenizer(reader, this.type, this.filter);
		return new TokenStreamComponents(tokenizer);
	}

	/**
	 * 获得一个tokenizer
	 * 
	 * @param reader
	 * @param type
	 * @param filter
	 * @return
	 */
	public static Tokenizer getTokenizer(BufferedReader reader, TYPE type, Set<String> filter) {
		Tokenizer tokenizer;

		switch (type) {
		case index:
			tokenizer = new AnsjTokenizer(new IndexAnalysis(reader), filter);
			break;
		case dic:
		case user:
			tokenizer = new AnsjTokenizer(new DicAnalysis(reader), filter);
			break;

		case to:
		case query:
		case search:
			tokenizer = new AnsjTokenizer(new ToAnalysis(reader), filter);
			break;
		default:
			tokenizer = new AnsjTokenizer(new ToAnalysis(reader), filter);
		}

		return tokenizer;
	}
}
{'fraction_non_alphanumeric': 0.11186113789778207, 'fraction_numerical': 0.0019286403085824494, 'mean_word_length': 3.934348239771646, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2146046', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1758, 'n_words': 402}