<commit_before>/*
 * <author><NAME></author>
 * <email><EMAIL></email>
 * <create-date>2018-06-28 7:37 PM</create-date>
 *
 * <copyright file="LogLinearModel.java">
 * Copyright (c) 2018, <NAME>. All Rights Reserved, http://www.hankcs.com/
 * This source is subject to Han He. Please contact Han He for more information.
 * </copyright>
 */
package com.hankcs.hanlp.model.crf;

import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.corpus.io.ByteArray;
import com.hankcs.hanlp.corpus.io.FileIOAdapter;
import com.hankcs.hanlp.corpus.io.IOUtil;
import com.hankcs.hanlp.model.perceptron.common.TaskType;
import com.hankcs.hanlp.model.perceptron.feature.FeatureMap;
import com.hankcs.hanlp.model.perceptron.feature.MutableFeatureMap;
import com.hankcs.hanlp.model.perceptron.model.LinearModel;
import com.hankcs.hanlp.model.perceptron.tagset.CWSTagSet;
import com.hankcs.hanlp.model.perceptron.tagset.NERTagSet;
import com.hankcs.hanlp.model.perceptron.tagset.TagSet;
import com.hankcs.hanlp.utility.Predefine;

import java.io.DataOutputStream;
import java.io.IOException;
import java.util.*;

import static com.hankcs.hanlp.utility.Predefine.BIN_EXT;
import static com.hankcs.hanlp.utility.Predefine.logger;

/**
 * 对数线性模型形式的CRF模型
 *
 * @author hankcs
 */
public class LogLinearModel extends LinearModel
{
    /**
     * 特征模板
     */
    private FeatureTemplate[] featureTemplateArray;

    private LogLinearModel(FeatureMap featureMap, float[] parameter)
    {
        super(featureMap, parameter);
    }

    private LogLinearModel(FeatureMap featureMap)
    {
        super(featureMap);
    }

    @Override
    public boolean load(ByteArray byteArray)
    {
        if (!super.load(byteArray)) return false;
        int size = byteArray.nextInt();
        featureTemplateArray = new FeatureTemplate[size];
        for (int i = 0; i < size; ++i)
        {
            FeatureTemplate featureTemplate = new FeatureTemplate();
            featureTemplate.load(byteArray);
            featureTemplateArray[i] = featureTemplate;
        }
        if (!byteArray.hasMore())
            byteArray.close();
        return true;
    }

    /**
     * 加载CRF模型
     *
     * @param modelFile HanLP的.bin格式，或CRF++的.txt格式（将会自动转换为model.txt.bin，下次会直接加载.txt.bin）
     * @throws IOException
     */
    public LogLinearModel(String modelFile) throws IOException
    {
        super(null, null);
        if (modelFile.endsWith(BIN_EXT))
        {
            load(modelFile); // model.bin
            return;
        }
        String binPath = modelFile + Predefine.BIN_EXT;

        if (!((HanLP.Config.IOAdapter == null || HanLP.Config.IOAdapter instanceof FileIOAdapter) && !IOUtil.isFileExisted(binPath)))
        {
            try
            {
                load(binPath); // model.txt -> model.bin
                return;
            }
            catch (Exception e)
            {
                // ignore
            }
        }

        convert(modelFile, binPath);
    }

    /**
     * 加载txt，转换为bin
     *
     * @param txtFile txt
     * @param binFile bin
     * @throws IOException
     */
    public LogLinearModel(String txtFile, String binFile) throws IOException
    {
        super(null, null);
        convert(txtFile, binFile);
    }

    private void convert(String txtFile, String binFile) throws IOException
    {
        TagSet tagSet = new TagSet(TaskType.CLASSIFICATION);
        IOUtil.LineIterator lineIterator = new IOUtil.LineIterator(txtFile);
        if (!lineIterator.hasNext()) throw new IOException("空白文件");
        logger.info(lineIterator.next());   // verson
        logger.info(lineIterator.next());   // cost-factor
        int maxid = Integer.parseInt(lineIterator.next().substring("maxid:".length()).trim());
        logger.info(lineIterator.next());   // xsize
        lineIterator.next();    // blank
        String line;
        while ((line = lineIterator.next()).length() != 0)
        {
            tagSet.add(line);
        }
        tagSet.type = guessModelType(tagSet);
        switch (tagSet.type)
        {
            case CWS:
                tagSet = new CWSTagSet(tagSet.idOf("B"), tagSet.idOf("M"), tagSet.idOf("E"), tagSet.idOf("S"));
                break;
            case NER:
                tagSet = new NERTagSet(tagSet.idOf("O"), tagSet.tags());
                break;
        }
        this.featureMap = new MutableFeatureMap(tagSet);
        FeatureMap featureMap = this.featureMap;
        final int sizeOfTagSet = tagSet.size();
        TreeMap<String, FeatureFunction> featureFunctionMap = new TreeMap<String, FeatureFunction>();  // 构建trie树的时候用
        TreeMap<Integer, FeatureFunction> featureFunctionList = new TreeMap<Integer, FeatureFunction>(); // 读取权值的时候用
        ArrayList<FeatureTemplate> featureTemplateList = new ArrayList<FeatureTemplate>();
        float[][] matrix = null;
        while ((line = lineIterator.next()).length() != 0)
        {
            if (!"B".equals(line))
            {
                FeatureTemplate featureTemplate = FeatureTemplate.create(line);
                featureTemplateList.add(featureTemplate);
            }
            else
            {
                matrix = new float[sizeOfTagSet][sizeOfTagSet];
            }
        }
        this.featureTemplateArray = featureTemplateList.toArray(new FeatureTemplate[0]);

        int b = -1;// 转换矩阵的权重位置
        if (matrix != null)
        {
            String[] args = lineIterator.next().split(" ", 2);    // 0 B
            b = Integer.valueOf(args[0]);<commit_msg>修正CRF++模型转换<commit_after>/*
 * <author><NAME></author>
 * <email><EMAIL></email>
 * <create-date>2018-06-28 7:37 PM</create-date>
 *
 * <copyright file="LogLinearModel.java">
 * Copyright (c) 2018, <NAME>. All Rights Reserved, http://www.hankcs.com/
 * This source is subject to Han He. Please contact Han He for more information.
 * </copyright>
 */
package com.hankcs.hanlp.model.crf;

import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.corpus.io.ByteArray;
import com.hankcs.hanlp.corpus.io.FileIOAdapter;
import com.hankcs.hanlp.corpus.io.IOUtil;
import com.hankcs.hanlp.model.perceptron.common.TaskType;
import com.hankcs.hanlp.model.perceptron.feature.FeatureMap;
import com.hankcs.hanlp.model.perceptron.feature.MutableFeatureMap;
import com.hankcs.hanlp.model.perceptron.model.LinearModel;
import com.hankcs.hanlp.model.perceptron.tagset.CWSTagSet;
import com.hankcs.hanlp.model.perceptron.tagset.NERTagSet;
import com.hankcs.hanlp.model.perceptron.tagset.TagSet;
import com.hankcs.hanlp.utility.Predefine;

import java.io.DataOutputStream;
import java.io.IOException;
import java.util.*;

import static com.hankcs.hanlp.utility.Predefine.BIN_EXT;
import static com.hankcs.hanlp.utility.Predefine.logger;

/**
 * 对数线性模型形式的CRF模型
 *
 * @author hankcs
 */
public class LogLinearModel extends LinearModel
{
    /**
     * 特征模板
     */
    private FeatureTemplate[] featureTemplateArray;

    private LogLinearModel(FeatureMap featureMap, float[] parameter)
    {
        super(featureMap, parameter);
    }

    private LogLinearModel(FeatureMap featureMap)
    {
        super(featureMap);
    }

    @Override
    public boolean load(ByteArray byteArray)
    {
        if (!super.load(byteArray)) return false;
        int size = byteArray.nextInt();
        featureTemplateArray = new FeatureTemplate[size];
        for (int i = 0; i < size; ++i)
        {
            FeatureTemplate featureTemplate = new FeatureTemplate();
            featureTemplate.load(byteArray);
            featureTemplateArray[i] = featureTemplate;
        }
        if (!byteArray.hasMore())
            byteArray.close();
        return true;
    }

    /**
     * 加载CRF模型
     *
     * @param modelFile HanLP的.bin格式，或CRF++的.txt格式（将会自动转换为model.txt.bin，下次会直接加载.txt.bin）
     * @throws IOException
     */
    public LogLinearModel(String modelFile) throws IOException
    {
        super(null, null);
        if (modelFile.endsWith(BIN_EXT))
        {
            load(modelFile); // model.bin
            return;
        }
        String binPath = modelFile + Predefine.BIN_EXT;

        if (!((HanLP.Config.IOAdapter == null || HanLP.Config.IOAdapter instanceof FileIOAdapter) && !IOUtil.isFileExisted(binPath)))
        {
            try
            {
                load(binPath); // model.txt -> model.bin
                return;
            }
            catch (Exception e)
            {
                // ignore
            }
        }

        convert(modelFile, binPath);
    }

    /**
     * 加载txt，转换为bin
     *
     * @param txtFile txt
     * @param binFile bin
     * @throws IOException
     */
    public LogLinearModel(String txtFile, String binFile) throws IOException
    {
        super(null, null);
        convert(txtFile, binFile);
    }

    private void convert(String txtFile, String binFile) throws IOException
    {
        TagSet tagSet = new TagSet(TaskType.CLASSIFICATION);
        IOUtil.LineIterator lineIterator = new IOUtil.LineIterator(txtFile);
        if (!lineIterator.hasNext()) throw new IOException("空白文件");
        logger.info(lineIterator.next());   // verson
        logger.info(lineIterator.next());   // cost-factor
        int maxid = Integer.parseInt(lineIterator.next().substring("maxid:".length()).trim());
        logger.info(lineIterator.next());   // xsize
        lineIterator.next();    // blank
        String line;
        while ((line = lineIterator.next()).length() != 0)
        {
            tagSet.add(line);
        }
        tagSet.type = guessModelType(tagSet);
        switch (tagSet.type)
        {
            case CWS:
                tagSet = new CWSTagSet(tagSet.idOf("B"), tagSet.idOf("M"), tagSet.idOf("E"), tagSet.idOf("S"));
                break;
            case NER:
                tagSet = new NERTagSet(tagSet.idOf("O"), tagSet.tags());
                break;
        }
        tagSet.lock();
        this.featureMap = new MutableFeatureMap(tagSet);
        FeatureMap featureMap = this.featureMap;
        final int sizeOfTagSet = tagSet.size();
        TreeMap<String, FeatureFunction> featureFunctionMap = new TreeMap<String, FeatureFunction>();  // 构建trie树的时候用
        TreeMap<Integer, FeatureFunction> featureFunctionList = new TreeMap<Integer, FeatureFunction>(); // 读取权值的时候用
        ArrayList<FeatureTemplate> featureTemplateList = new ArrayList<FeatureTemplate>();
        float[][] matrix = null;
        while ((line = lineIterator.next()).length() != 0)
        {
            if (!"B".equals(line))
            {
                FeatureTemplate featureTemplate = FeatureTemplate.create(line);
                featureTemplateList.add(featureTemplate);
            }
            else
            {
                matrix = new float[sizeOfTagSet][sizeOfTagSet];
            }
        }
        this.featureTemplateArray = featureTemplateList.toArray(new FeatureTemplate[0]);

        int b = -1;// 转换矩阵的权重位置
        if (matrix != null)
        {
            String[] args = lineIterator.next().split(" ", 2);    // 0 B
            b = Integer.valueOf(args[0]);
{'fraction_non_alphanumeric': 0.12074107546317217, 'fraction_numerical': 0.004157252598282874, 'mean_word_length': 2.3421926910299002, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5136674', 'n_tokens_mistral': 3430, 'n_tokens_neox': 3400, 'n_words': 616}