<commit_before>package com.chenlb.mmseg4j;

import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.plugin.analysis.mmseg.AnalysisMMsegPlugin;

import java.io.*;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

/**
 * 词典类. 词库目录单例模式.<br/>
 * 保存单字与其频率,还有词库.<br/>
 * 有检测词典变更的接口，外部程序可以使用 {@link #wordsFileIsChange()} 和 {@link #reload()} 来完成检测与加载的工作.
 *
 * @author chenlb 2009-2-20 下午11:34:29
 */
public class Dictionary {

	private static final ESLogger log = Loggers.getLogger("mmseg-analyzer");

	private File dicPath;	//词库目录
	private volatile Map<Character, CharNode> dict;
	private volatile Map<Character, Object> unit;	//单个字的单位

	/** 记录 word 文件的最后修改时间 */
	private Map<File, Long> wordsLastTime = null;
	private long lastLoadTime = 0;

	/** 不要直接使用, 通过 {@link #getDefalutPath()} 使用*/
	private static File defalutPath = null;
	private static final ConcurrentHashMap<File, Dictionary> dics = new ConcurrentHashMap<File, Dictionary>();

    /**
     * 词典的目录
     */
    private Dictionary(File path) {
        init(path);
    }

	protected void finalize() throws Throwable {
		/*
		 * 使 class reload 的时也可以释放词库
		 */
		destroy();
	}

	/**
	 * 从默认目录加载词库文件.<p/>
	 * 查找默认目录顺序:
	 * <ol>
	 * <li>从系统属性mmseg.dic.path指定的目录中加载</li>
	 * <li>从classpath/data目录</li>
	 * <li>从user.dir/data目录</li>
	 * </ol>
	 * @see #getDefalutPath()
	 */
	public static Dictionary getInstance() {
		Path path = PathUtils.get(getDictRoot(), "mmseg");
		return getInstance(path.toFile());
	}

	/**
	 * @param path 词典的目录
	 */
	public static Dictionary getInstance(String path) {
		return getInstance(new File(path));
	}

	/**
	 * @param path 词典的目录
	 */
	public static Dictionary getInstance(File path) {
		File normalizeDir = normalizeFile(path);
		Dictionary dic = dics.get(normalizeDir);
		if(dic == null) {
			dic = new Dictionary(normalizeDir);
			dics.put(normalizeDir, dic);
		}
		return dic;
	}

	public static File normalizeFile(File file) {
		if(file == defalutPath) {
			return defalutPath;
		}
		try {
			return file.getCanonicalFile();
		} catch (IOException e) {
			throw new RuntimeException("normalize file=["+file+"] fail", e);
		}
	}

	/**
	 * 销毁, 释放资源. 此后此对像不再可用.
	 */
	void destroy() {
		clear(dicPath);

		dicPath = null;
		dict = null;
		unit = null;
	}

	/**
	 * @see Dictionary#clear(File)
	 */
	public static Dictionary clear(String path) {
		return clear(new File(path));
	}

	/**
	 * 从单例缓存中去除
	 * @param path
	 * @return 没有返回 null
	 */
	public static Dictionary clear(File path) {
		File normalizeDir = normalizeFile(path);
		return dics.remove(normalizeDir);
	}

	private void init(File path) {
		dicPath = path;
		wordsLastTime = new HashMap<File, Long>();

		reload();	//加载词典
	}

	private static long now() {
		return System.currentTimeMillis();
	}

	/**
	 * 只要 wordsXXX.dic的文件
	 * @return
	 */
	protected File[] listWordsFiles() {
		return dicPath.listFiles(new FilenameFilter() {

			public boolean accept(File dir, String name) {

				return name.startsWith("words") && name.endsWith(".dic");
			}

		});
	}

	private Map<Character, CharNode> loadDic(File wordsPath) throws IOException {
		InputStream charsIn;
		File charsFile = new File(wordsPath, "chars.dic");
		System.out.println("ERROR:"+charsFile.getAbsolutePath());
		if(charsFile.exists()) {
			charsIn = new FileInputStream(charsFile);
			addLastTime(charsFile);	//chars.dic 也检测是否变更
		} else {	//从 jar 里加载
			charsIn = this.getClass().getResourceAsStream("/data/chars.dic");
			charsFile = new File(this.getClass().getResource("/data/chars.dic").getFile());	//only for log
		}
		final Map<Character, CharNode> dic = new HashMap<Character, CharNode>();
		int lineNum;
		long s = now();
		long ss = s;
		lineNum = load(charsIn, new FileLoading() {	//单个字的

			public void row(String line, int n) {
				if(line.length() < 1) {
					return;
				}
				String[] w = line.split(" ");
				CharNode cn = new CharNode();
				switch(w.length) {<commit_msg>clean debug info<commit_after>package com.chenlb.mmseg4j;

import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.plugin.analysis.mmseg.AnalysisMMsegPlugin;

import java.io.*;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

/**
 * 词典类. 词库目录单例模式.<br/>
 * 保存单字与其频率,还有词库.<br/>
 * 有检测词典变更的接口，外部程序可以使用 {@link #wordsFileIsChange()} 和 {@link #reload()} 来完成检测与加载的工作.
 *
 * @author chenlb 2009-2-20 下午11:34:29
 */
public class Dictionary {

	private static final ESLogger log = Loggers.getLogger("mmseg-analyzer");

	private File dicPath;	//词库目录
	private volatile Map<Character, CharNode> dict;
	private volatile Map<Character, Object> unit;	//单个字的单位

	/** 记录 word 文件的最后修改时间 */
	private Map<File, Long> wordsLastTime = null;
	private long lastLoadTime = 0;

	/** 不要直接使用, 通过 {@link #getDefalutPath()} 使用*/
	private static File defalutPath = null;
	private static final ConcurrentHashMap<File, Dictionary> dics = new ConcurrentHashMap<File, Dictionary>();

    /**
     * 词典的目录
     */
    private Dictionary(File path) {
        init(path);
    }

	protected void finalize() throws Throwable {
		/*
		 * 使 class reload 的时也可以释放词库
		 */
		destroy();
	}

	/**
	 * 从默认目录加载词库文件.<p/>
	 * 查找默认目录顺序:
	 * <ol>
	 * <li>从系统属性mmseg.dic.path指定的目录中加载</li>
	 * <li>从classpath/data目录</li>
	 * <li>从user.dir/data目录</li>
	 * </ol>
	 * @see #getDefalutPath()
	 */
	public static Dictionary getInstance() {
		Path path = PathUtils.get(getDictRoot(), "mmseg");
		return getInstance(path.toFile());
	}

	/**
	 * @param path 词典的目录
	 */
	public static Dictionary getInstance(String path) {
		return getInstance(new File(path));
	}

	/**
	 * @param path 词典的目录
	 */
	public static Dictionary getInstance(File path) {
		File normalizeDir = normalizeFile(path);
		Dictionary dic = dics.get(normalizeDir);
		if(dic == null) {
			dic = new Dictionary(normalizeDir);
			dics.put(normalizeDir, dic);
		}
		return dic;
	}

	public static File normalizeFile(File file) {
		if(file == defalutPath) {
			return defalutPath;
		}
		try {
			return file.getCanonicalFile();
		} catch (IOException e) {
			throw new RuntimeException("normalize file=["+file+"] fail", e);
		}
	}

	/**
	 * 销毁, 释放资源. 此后此对像不再可用.
	 */
	void destroy() {
		clear(dicPath);

		dicPath = null;
		dict = null;
		unit = null;
	}

	/**
	 * @see Dictionary#clear(File)
	 */
	public static Dictionary clear(String path) {
		return clear(new File(path));
	}

	/**
	 * 从单例缓存中去除
	 * @param path
	 * @return 没有返回 null
	 */
	public static Dictionary clear(File path) {
		File normalizeDir = normalizeFile(path);
		return dics.remove(normalizeDir);
	}

	private void init(File path) {
		dicPath = path;
		wordsLastTime = new HashMap<File, Long>();

		reload();	//加载词典
	}

	private static long now() {
		return System.currentTimeMillis();
	}

	/**
	 * 只要 wordsXXX.dic的文件
	 * @return
	 */
	protected File[] listWordsFiles() {
		return dicPath.listFiles(new FilenameFilter() {

			public boolean accept(File dir, String name) {

				return name.startsWith("words") && name.endsWith(".dic");
			}

		});
	}

	private Map<Character, CharNode> loadDic(File wordsPath) throws IOException {
		InputStream charsIn;
		File charsFile = new File(wordsPath, "chars.dic");
		if(charsFile.exists()) {
			charsIn = new FileInputStream(charsFile);
			addLastTime(charsFile);	//chars.dic 也检测是否变更
		} else {	//从 jar 里加载
			charsIn = this.getClass().getResourceAsStream("/data/chars.dic");
			charsFile = new File(this.getClass().getResource("/data/chars.dic").getFile());	//only for log
		}
		final Map<Character, CharNode> dic = new HashMap<Character, CharNode>();
		int lineNum;
		long s = now();
		long ss = s;
		lineNum = load(charsIn, new FileLoading() {	//单个字的

			public void row(String line, int n) {
				if(line.length() < 1) {
					return;
				}
				String[] w = line.split(" ");
				CharNode cn = new CharNode();
				switch(w.length) {
{'fraction_non_alphanumeric': 0.142805625676163, 'fraction_numerical': 0.0038466161798293066, 'mean_word_length': 3.703222159412097, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2360931', 'n_tokens_mistral': 3346, 'n_tokens_neox': 3192, 'n_words': 663}