<commit_before>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electric Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */

package edu.cmu.sphinx.linguist.language.ngram.large;

import edu.cmu.sphinx.linguist.WordSequence;
import edu.cmu.sphinx.linguist.dictionary.Dictionary;
import edu.cmu.sphinx.linguist.dictionary.Word;
import edu.cmu.sphinx.linguist.language.ngram.LanguageModel;
import edu.cmu.sphinx.util.LogMath;
import edu.cmu.sphinx.util.TimerPool;
import edu.cmu.sphinx.util.props.*;

import java.io.*;
import java.net.URL;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * Queries a binary language model file generated by the <a href="http://www.speech.cs.cmu.edu/SLM_info.html">
 * CMU-Cambridge Statistical Language Modeling Toolkit</a>.
 * <p/>
 * Note that all probabilities in the grammar are stored in LogMath log base format. Language Probabilities in the
 * language model file are stored in log 10 base. They are converted to the LogMath logbase.
 */
public class LargeTrigramModel implements LanguageModel {

    /**
     * A property for the name of the file that logs all the queried N-grams. If this property is set to null, it
     * means that the queried N-grams are not logged.
     */
    @S4String(mandatory = false)
    public static final String PROP_QUERY_LOG_FILE = "queryLogFile";

    /** A property that defines that maxium number of trigrams to be cached */
    @S4Integer(defaultValue = 100000)
    public static final String PROP_TRIGRAM_CACHE_SIZE = "trigramCacheSize";

    /** A property that defines the maximum number of bigrams to be cached. */
    @S4Integer(defaultValue = 50000)
    public static final String PROP_BIGRAM_CACHE_SIZE = "bigramCacheSize";

    /** A property that controls whether the bigram and trigram caches are cleared after every utterance */
    @S4Boolean(defaultValue = false)
    public static final String PROP_CLEAR_CACHES_AFTER_UTTERANCE = "clearCachesAfterUtterance";

    /** A property that defines the language weight for the search */
    @S4Double(defaultValue = 1.0)
    public final static String PROP_LANGUAGE_WEIGHT = "languageWeight";

    /** A property that defines the logMath component. */
    @S4Component(type = LogMath.class)
    public final static String PROP_LOG_MATH = "logMath";

    /**
     * A property that controls whether or not the language model will apply the language weight and word insertion
     * probability
     */
    @S4Boolean(defaultValue = false)
    public final static String PROP_APPLY_LANGUAGE_WEIGHT_AND_WIP = "applyLanguageWeightAndWip";

    /** Word insertion probability property */
    @S4Double(defaultValue = 1.0)
    public final static String PROP_WORD_INSERTION_PROBABILITY = "wordInsertionProbability";

    /** If true, use full bigram information to determine smear */
    @S4Boolean(defaultValue = false)
    public final static String PROP_FULL_SMEAR = "fullSmear";

    /**
     * The number of bytes per bigram in the LM file generated by the CMU-Cambridge Statistical Language Modelling
     * Toolkit.
     */
    public static final int BYTES_PER_BIGRAM = 8;

    /**
     * The number of bytes per trigram in the LM file generated by the CMU-Cambridge Statistical Language Modelling
     * Toolkit.
     */
    public static final int BYTES_PER_TRIGRAM = 4;

    private final static int SMEAR_MAGIC = 0xC0CAC01A; // things go better 

    public Logger getLogger() {
        return logger;
    }


    // ------------------------------
    // Configuration data
    // ------------------------------
    private Logger logger;
    private LogMath logMath;
    private String name;
    private String ngramLogFile;
    private int maxTrigramCacheSize;
    private int maxBigramCacheSize;
    private boolean clearCacheAfterUtterance;
    private boolean fullSmear;
    private int maxDepth;
    private Dictionary dictionary;
    private String format;
    private File location;
    private boolean applyLanguageWeightAndWip;
    private float languageWeight;
    private double wip;
    private float unigramWeight;

    // -------------------------------
    // Statistics
    // -------------------------------
    private int bigramMisses;
    private int trigramMisses;
    private int trigramHit;
    private int smearTermCount = 0;

    // -------------------------------
    // subcomponents
    // --------------------------------
    private BinaryLoader loader;
    private PrintWriter logFile;

    // -------------------------------
    // Working data
    // --------------------------------
    private Map<Word, UnigramProbability> unigramIDMap;
    private Map<WordSequence, TrigramBuffer> loadedTrigramBuffer;
    private LRUCache<WordSequence, Float> trigramCache;
    private LRUCache<WordSequence, BigramProbability> bigramCache;
    private Map<Long, Float> bigramSmearMap;

    private BigramBuffer[] loadedBigramBuffers;
    private UnigramProbability[] unigrams;
    private int[] trigramSegmentTable;
    private float[] bigramProbTable;
    private float[] trigramProbTable;
    private float[] trigramBackoffTable;
    private float[] unigramSmearTerm;


    /*
    * (non-Javadoc)
    *
    * @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
    */
    public void newProperties(PropertySheet ps) throws PropertyException {
        logger = ps.getLogger();
        format = ps.getString(LanguageModel.PROP_FORMAT
        );
             
        URL urlLocation = ConfigurationManagerUtils.getResource(PROP_LOCATION, ps);
        location = new File (urlLocation.getFile());
        
        ngramLogFile = ps.getString(PROP_QUERY_LOG_FILE
        );
        maxTrigramCacheSize = ps.getInt(PROP_TRIGRAM_CACHE_SIZE
        );
        maxBigramCacheSize = ps.getInt(PROP_BIGRAM_CACHE_SIZE
        );
        clearCacheAfterUtterance = ps.getBoolean(
                PROP_CLEAR_CACHES_AFTER_UTTERANCE
        );
        maxDepth = ps.getInt(LanguageModel.PROP_MAX_DEPTH
        );
        logMath = (LogMath) ps.getComponent(PROP_LOG_MATH);
        dictionary = (Dictionary) ps.getComponent(PROP_DICTIONARY
        );
        applyLanguageWeightAndWip = ps.getBoolean(
                PROP_APPLY_LANGUAGE_WEIGHT_AND_WIP
        );
        languageWeight = ps.getFloat(PROP_LANGUAGE_WEIGHT
        );
        wip = ps.getDouble(PROP_WORD_INSERTION_PROBABILITY
        );
        unigramWeight = ps.getFloat(PROP_UNIGRAM_WEIGHT
        );

        fullSmear = ps.getBoolean(PROP_FULL_SMEAR);
    }


    /*
    * (non-Javadoc)
    *
    * @see edu.cmu.sphinx.util.props.Configurable#getName()
    */
    public String getName() {
        return name;
    }


    /*
    * (non-Javadoc)
    *
    * @see edu.cmu.sphinx.linguist.language.ngram.LanguageModel#allocate()
    */
    public void allocate() throws IOException {
        TimerPool.getTimer(this, "LM Load").start();
        // create the log file if specified
        if (ngramLogFile != null) {
            logFile = new PrintWriter(new FileOutputStream(ngramLogFile));
        }
        unigramIDMap = new HashMap<Word, UnigramProbability>();
        loadedTrigramBuffer = new HashMap<WordSequence, TrigramBuffer>();
        trigramCache = new LRUCache<WordSequence, Float>(maxTrigramCacheSize);
        bigramCache = new LRUCache<WordSequence, BigramProbability>(maxBigramCacheSize);
        loader = new BinaryLoader(format, location, applyLanguageWeightAndWip,
                logMath, languageWeight, wip, unigramWeight);
        unigrams = loader.getUnigrams();
        bigramProbTable = loader.getBigramProbabilities();
        trigramProbTable = loader.getTrigramProbabilities();
        trigramBackoffTable = loader.getTrigramBackoffWeights();
        trigramSegmentTable = loader.getTrigramSegments();
        buildUnigramIDMap(dictionary);
        loadedBigramBuffers = new BigramBuffer[unigrams.length];

	if (maxDepth > loader.getMaxDepth()) {
            maxDepth = loader.getMaxDepth();
        } else if (maxDepth == 0) {
            throw new Error("Invalid LM max-depth: " + maxDepth);
        }

        logger.info("Unigrams: " + loader.getNumberUnigrams());
        logger.info("Bigrams: " + loader.getNumberBigrams());
        logger.info("Trigrams: " + loader.getNumberTrigrams());

        if (fullSmear) {
            System.out.println("Full Smear");
            try {<commit_msg>Fixed the handling of the default value of MaxDepth<commit_after>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electric Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */

package edu.cmu.sphinx.linguist.language.ngram.large;

import edu.cmu.sphinx.linguist.WordSequence;
import edu.cmu.sphinx.linguist.dictionary.Dictionary;
import edu.cmu.sphinx.linguist.dictionary.Word;
import edu.cmu.sphinx.linguist.language.ngram.LanguageModel;
import edu.cmu.sphinx.util.LogMath;
import edu.cmu.sphinx.util.TimerPool;
import edu.cmu.sphinx.util.props.*;

import java.io.*;
import java.net.URL;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * Queries a binary language model file generated by the <a href="http://www.speech.cs.cmu.edu/SLM_info.html">
 * CMU-Cambridge Statistical Language Modeling Toolkit</a>.
 * <p/>
 * Note that all probabilities in the grammar are stored in LogMath log base format. Language Probabilities in the
 * language model file are stored in log 10 base. They are converted to the LogMath logbase.
 */
public class LargeTrigramModel implements LanguageModel {

    /**
     * A property for the name of the file that logs all the queried N-grams. If this property is set to null, it
     * means that the queried N-grams are not logged.
     */
    @S4String(mandatory = false)
    public static final String PROP_QUERY_LOG_FILE = "queryLogFile";

    /** A property that defines that maxium number of trigrams to be cached */
    @S4Integer(defaultValue = 100000)
    public static final String PROP_TRIGRAM_CACHE_SIZE = "trigramCacheSize";

    /** A property that defines the maximum number of bigrams to be cached. */
    @S4Integer(defaultValue = 50000)
    public static final String PROP_BIGRAM_CACHE_SIZE = "bigramCacheSize";

    /** A property that controls whether the bigram and trigram caches are cleared after every utterance */
    @S4Boolean(defaultValue = false)
    public static final String PROP_CLEAR_CACHES_AFTER_UTTERANCE = "clearCachesAfterUtterance";

    /** A property that defines the language weight for the search */
    @S4Double(defaultValue = 1.0)
    public final static String PROP_LANGUAGE_WEIGHT = "languageWeight";

    /** A property that defines the logMath component. */
    @S4Component(type = LogMath.class)
    public final static String PROP_LOG_MATH = "logMath";

    /**
     * A property that controls whether or not the language model will apply the language weight and word insertion
     * probability
     */
    @S4Boolean(defaultValue = false)
    public final static String PROP_APPLY_LANGUAGE_WEIGHT_AND_WIP = "applyLanguageWeightAndWip";

    /** Word insertion probability property */
    @S4Double(defaultValue = 1.0)
    public final static String PROP_WORD_INSERTION_PROBABILITY = "wordInsertionProbability";

    /** If true, use full bigram information to determine smear */
    @S4Boolean(defaultValue = false)
    public final static String PROP_FULL_SMEAR = "fullSmear";

    /**
     * The number of bytes per bigram in the LM file generated by the CMU-Cambridge Statistical Language Modelling
     * Toolkit.
     */
    public static final int BYTES_PER_BIGRAM = 8;

    /**
     * The number of bytes per trigram in the LM file generated by the CMU-Cambridge Statistical Language Modelling
     * Toolkit.
     */
    public static final int BYTES_PER_TRIGRAM = 4;

    private final static int SMEAR_MAGIC = 0xC0CAC01A; // things go better 

    public Logger getLogger() {
        return logger;
    }


    // ------------------------------
    // Configuration data
    // ------------------------------
    private Logger logger;
    private LogMath logMath;
    private String name;
    private String ngramLogFile;
    private int maxTrigramCacheSize;
    private int maxBigramCacheSize;
    private boolean clearCacheAfterUtterance;
    private boolean fullSmear;
    private int maxDepth;
    private Dictionary dictionary;
    private String format;
    private File location;
    private boolean applyLanguageWeightAndWip;
    private float languageWeight;
    private double wip;
    private float unigramWeight;

    // -------------------------------
    // Statistics
    // -------------------------------
    private int bigramMisses;
    private int trigramMisses;
    private int trigramHit;
    private int smearTermCount = 0;

    // -------------------------------
    // subcomponents
    // --------------------------------
    private BinaryLoader loader;
    private PrintWriter logFile;

    // -------------------------------
    // Working data
    // --------------------------------
    private Map<Word, UnigramProbability> unigramIDMap;
    private Map<WordSequence, TrigramBuffer> loadedTrigramBuffer;
    private LRUCache<WordSequence, Float> trigramCache;
    private LRUCache<WordSequence, BigramProbability> bigramCache;
    private Map<Long, Float> bigramSmearMap;

    private BigramBuffer[] loadedBigramBuffers;
    private UnigramProbability[] unigrams;
    private int[] trigramSegmentTable;
    private float[] bigramProbTable;
    private float[] trigramProbTable;
    private float[] trigramBackoffTable;
    private float[] unigramSmearTerm;


    /*
    * (non-Javadoc)
    *
    * @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
    */
    public void newProperties(PropertySheet ps) throws PropertyException {
        logger = ps.getLogger();
        format = ps.getString(LanguageModel.PROP_FORMAT
        );
             
        URL urlLocation = ConfigurationManagerUtils.getResource(PROP_LOCATION, ps);
        location = new File (urlLocation.getFile());
        
        ngramLogFile = ps.getString(PROP_QUERY_LOG_FILE
        );
        maxTrigramCacheSize = ps.getInt(PROP_TRIGRAM_CACHE_SIZE
        );
        maxBigramCacheSize = ps.getInt(PROP_BIGRAM_CACHE_SIZE
        );
        clearCacheAfterUtterance = ps.getBoolean(
                PROP_CLEAR_CACHES_AFTER_UTTERANCE
        );
        maxDepth = ps.getInt(LanguageModel.PROP_MAX_DEPTH
        );
        logMath = (LogMath) ps.getComponent(PROP_LOG_MATH);
        dictionary = (Dictionary) ps.getComponent(PROP_DICTIONARY
        );
        applyLanguageWeightAndWip = ps.getBoolean(
                PROP_APPLY_LANGUAGE_WEIGHT_AND_WIP
        );
        languageWeight = ps.getFloat(PROP_LANGUAGE_WEIGHT
        );
        wip = ps.getDouble(PROP_WORD_INSERTION_PROBABILITY
        );
        unigramWeight = ps.getFloat(PROP_UNIGRAM_WEIGHT
        );

        fullSmear = ps.getBoolean(PROP_FULL_SMEAR);
    }


    /*
    * (non-Javadoc)
    *
    * @see edu.cmu.sphinx.util.props.Configurable#getName()
    */
    public String getName() {
        return name;
    }


    /*
    * (non-Javadoc)
    *
    * @see edu.cmu.sphinx.linguist.language.ngram.LanguageModel#allocate()
    */
    public void allocate() throws IOException {
        TimerPool.getTimer(this, "LM Load").start();
        // create the log file if specified
        if (ngramLogFile != null) {
            logFile = new PrintWriter(new FileOutputStream(ngramLogFile));
        }
        unigramIDMap = new HashMap<Word, UnigramProbability>();
        loadedTrigramBuffer = new HashMap<WordSequence, TrigramBuffer>();
        trigramCache = new LRUCache<WordSequence, Float>(maxTrigramCacheSize);
        bigramCache = new LRUCache<WordSequence, BigramProbability>(maxBigramCacheSize);
        loader = new BinaryLoader(format, location, applyLanguageWeightAndWip,
                logMath, languageWeight, wip, unigramWeight);
        unigrams = loader.getUnigrams();
        bigramProbTable = loader.getBigramProbabilities();
        trigramProbTable = loader.getTrigramProbabilities();
        trigramBackoffTable = loader.getTrigramBackoffWeights();
        trigramSegmentTable = loader.getTrigramSegments();
        buildUnigramIDMap(dictionary);
        loadedBigramBuffers = new BigramBuffer[unigrams.length];

	if (maxDepth <= 0 || maxDepth > loader.getMaxDepth()) {
            maxDepth = loader.getMaxDepth();
        }

        logger.info("Unigrams: " + loader.getNumberUnigrams());
        logger.info("Bigrams: " + loader.getNumberBigrams());
        logger.info("Trigrams: " + loader.getNumberTrigrams());

        if (fullSmear) {
            System.out.println("Full Smear");
            try {
{'fraction_non_alphanumeric': 0.10992805755395683, 'fraction_numerical': 0.0057553956834532375, 'mean_word_length': 2.7521053768084647, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2280134', 'n_tokens_mistral': 5227, 'n_tokens_neox': 4885, 'n_words': 1284}