<commit_before>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electronic Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */

package edu.cmu.sphinx.decoder.linguist.lextree;

import edu.cmu.sphinx.decoder.linguist.Linguist;
import edu.cmu.sphinx.decoder.linguist.Grammar;
import edu.cmu.sphinx.knowledge.dictionary.Dictionary;
import edu.cmu.sphinx.knowledge.dictionary.FullDictionary;
import edu.cmu.sphinx.decoder.linguist.SearchState;
import edu.cmu.sphinx.decoder.linguist.SearchStateArc;
import edu.cmu.sphinx.decoder.linguist.WordSearchState;
import edu.cmu.sphinx.decoder.linguist.UnitSearchState;
import edu.cmu.sphinx.decoder.linguist.HMMSearchState;
import edu.cmu.sphinx.decoder.linguist.util.LinguistTimer;

import edu.cmu.sphinx.knowledge.acoustic.HMM;
import edu.cmu.sphinx.knowledge.acoustic.HMMState;
import edu.cmu.sphinx.knowledge.acoustic.HMMStateArc;
import edu.cmu.sphinx.knowledge.acoustic.HMMPosition;
import edu.cmu.sphinx.knowledge.acoustic.Unit;
import edu.cmu.sphinx.knowledge.acoustic.AcousticModel;
import edu.cmu.sphinx.knowledge.acoustic.Context;
import edu.cmu.sphinx.knowledge.acoustic.LeftRightContext;
import edu.cmu.sphinx.knowledge.language.LanguageModel;
import edu.cmu.sphinx.knowledge.language.WordSequence;
import edu.cmu.sphinx.util.SphinxProperties;
import edu.cmu.sphinx.util.StatisticsVariable;
import edu.cmu.sphinx.util.LogMath;
import edu.cmu.sphinx.util.Timer;
import edu.cmu.sphinx.util.Utilities;
import edu.cmu.sphinx.knowledge.dictionary.Pronunciation;


import java.io.IOException;

import java.util.HashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.HashSet;
import java.util.Random;
import java.util.Comparator;

/**
 */
public class LexTreeLinguist implements  Linguist {

    private SphinxProperties props;

    private LanguageModel languageModel;
    private AcousticModel acousticModel;

    private LogMath logMath;
    private float languageWeight;
    private float logWordInsertionProbability;
    private float logSilenceInsertionProbability;
    private float logUnitInsertionProbability;

    // just for detailed debugging
    private final boolean tracing = false;


    private HMMPool hmmPool;
    private LexTree lexTree;
    private Dictionary dictionary;
    private final static SearchStateArc[] EMPTY_ARC = new SearchStateArc[0];
    private float logOne;
    private int silenceID;
    private LexTree.WordLexNode finalNode;
    

    /**
     * Creates a LexTree linguist associated with the given context
     *
     * @param context the context to associate this linguist with
     * @param languageModel the language model
     * @param grammar the grammar for this linguist
     * @param models the acoustic model used by this linguist
     */
    public void initialize(String context, LanguageModel languageModel,
                        Grammar grammar, AcousticModel[] models) {
        assert models.length == 1;

        this.props = SphinxProperties.getSphinxProperties(context);
        this.acousticModel = models[0];
        this.logMath = LogMath.getLogMath(context);
        this.languageModel = languageModel;


        try {
            this.dictionary = new FullDictionary(context);
        } catch (IOException ioe) {
            throw new Error("LexTreeLinguist: Can't load dictionary", ioe);
        }

        logOne = logMath.getLogOne();


        logWordInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_WORD_INSERTION_PROBABILITY,
              Linguist.PROP_WORD_INSERTION_PROBABILITY_DEFAULT));

        logSilenceInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_SILENCE_INSERTION_PROBABILITY,
              Linguist.PROP_SILENCE_INSERTION_PROBABILITY_DEFAULT));

        logUnitInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_UNIT_INSERTION_PROBABILITY,
              Linguist.PROP_UNIT_INSERTION_PROBABILITY_DEFAULT));

        languageWeight = props.getFloat(PROP_LANGUAGE_WEIGHT,
                                        PROP_LANGUAGE_WEIGHT_DEFAULT);


        compileGrammar();

        acousticModel = null;

        if (false) {
            LinguistTimer lt = new LinguistTimer(this, false);
            lt.timeLinguist(10, 500, 1000);
        }

        finalNode = lexTree.findWordNode(Dictionary.SENTENCE_END_SPELLING);
    }

    /**
     * 
     * Called before a recognition
     */
    public void start() {
	languageModel.start();
    }

    /**
     * Called after a recognition
     */
    public void stop() {
	languageModel.stop();
    }<commit_msg>Moved property getters to the 'start' routine. This allows changing some properties on the fly, which substantially reduces hand-tuning time.<commit_after>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electronic Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */

package edu.cmu.sphinx.decoder.linguist.lextree;

import edu.cmu.sphinx.decoder.linguist.Linguist;
import edu.cmu.sphinx.decoder.linguist.Grammar;
import edu.cmu.sphinx.knowledge.dictionary.Dictionary;
import edu.cmu.sphinx.knowledge.dictionary.FullDictionary;
import edu.cmu.sphinx.decoder.linguist.SearchState;
import edu.cmu.sphinx.decoder.linguist.SearchStateArc;
import edu.cmu.sphinx.decoder.linguist.WordSearchState;
import edu.cmu.sphinx.decoder.linguist.UnitSearchState;
import edu.cmu.sphinx.decoder.linguist.HMMSearchState;
import edu.cmu.sphinx.decoder.linguist.util.LinguistTimer;

import edu.cmu.sphinx.knowledge.acoustic.HMM;
import edu.cmu.sphinx.knowledge.acoustic.HMMState;
import edu.cmu.sphinx.knowledge.acoustic.HMMStateArc;
import edu.cmu.sphinx.knowledge.acoustic.HMMPosition;
import edu.cmu.sphinx.knowledge.acoustic.Unit;
import edu.cmu.sphinx.knowledge.acoustic.AcousticModel;
import edu.cmu.sphinx.knowledge.acoustic.Context;
import edu.cmu.sphinx.knowledge.acoustic.LeftRightContext;
import edu.cmu.sphinx.knowledge.language.LanguageModel;
import edu.cmu.sphinx.knowledge.language.WordSequence;
import edu.cmu.sphinx.util.SphinxProperties;
import edu.cmu.sphinx.util.StatisticsVariable;
import edu.cmu.sphinx.util.LogMath;
import edu.cmu.sphinx.util.Timer;
import edu.cmu.sphinx.util.Utilities;
import edu.cmu.sphinx.knowledge.dictionary.Pronunciation;


import java.io.IOException;

import java.util.HashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.HashSet;
import java.util.Random;
import java.util.Comparator;

/**
 */
public class LexTreeLinguist implements  Linguist {

    private SphinxProperties props;

    private LanguageModel languageModel;
    private AcousticModel acousticModel;

    private LogMath logMath;
    private float languageWeight;
    private float logWordInsertionProbability;
    private float logSilenceInsertionProbability;
    private float logUnitInsertionProbability;

    // just for detailed debugging
    private final boolean tracing = false;


    private HMMPool hmmPool;
    private LexTree lexTree;
    private Dictionary dictionary;
    private final static SearchStateArc[] EMPTY_ARC = new SearchStateArc[0];
    private float logOne;
    private int silenceID;
    private LexTree.WordLexNode finalNode;
    

    /**
     * Creates a LexTree linguist associated with the given context
     *
     * @param context the context to associate this linguist with
     * @param languageModel the language model
     * @param grammar the grammar for this linguist
     * @param models the acoustic model used by this linguist
     */
    public void initialize(String context, LanguageModel languageModel,
                        Grammar grammar, AcousticModel[] models) {
        assert models.length == 1;

        this.props = SphinxProperties.getSphinxProperties(context);
        this.acousticModel = models[0];
        this.logMath = LogMath.getLogMath(context);
        this.languageModel = languageModel;


        try {
            this.dictionary = new FullDictionary(context);
        } catch (IOException ioe) {
            throw new Error("LexTreeLinguist: Can't load dictionary", ioe);
        }

        logOne = logMath.getLogOne();


        logWordInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_WORD_INSERTION_PROBABILITY,
              Linguist.PROP_WORD_INSERTION_PROBABILITY_DEFAULT));

        logSilenceInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_SILENCE_INSERTION_PROBABILITY,
              Linguist.PROP_SILENCE_INSERTION_PROBABILITY_DEFAULT));

        logUnitInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_UNIT_INSERTION_PROBABILITY,
              Linguist.PROP_UNIT_INSERTION_PROBABILITY_DEFAULT));

        languageWeight = props.getFloat(PROP_LANGUAGE_WEIGHT,
                                        PROP_LANGUAGE_WEIGHT_DEFAULT);


        compileGrammar();

        acousticModel = null;

        if (false) {
            LinguistTimer lt = new LinguistTimer(this, false);
            lt.timeLinguist(10, 500, 1000);
        }

        finalNode = lexTree.findWordNode(Dictionary.SENTENCE_END_SPELLING);
    }

    /**
     * 
     * Called before a recognition
     */
    public void start() {
        // property getters are here in 'start' to allow changing
        // of these properties on the fly
        logWordInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_WORD_INSERTION_PROBABILITY,
              Linguist.PROP_WORD_INSERTION_PROBABILITY_DEFAULT));

        logSilenceInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_SILENCE_INSERTION_PROBABILITY,
              Linguist.PROP_SILENCE_INSERTION_PROBABILITY_DEFAULT));

        logUnitInsertionProbability = logMath.linearToLog
            (props.getDouble
             (Linguist.PROP_UNIT_INSERTION_PROBABILITY,
              Linguist.PROP_UNIT_INSERTION_PROBABILITY_DEFAULT));

        languageWeight = props.getFloat(PROP_LANGUAGE_WEIGHT,
                                        PROP_LANGUAGE_WEIGHT_DEFAULT);

	languageModel.start();
    }

    /**
     * Called after a recognition
     */
    public void stop() {
	languageModel.stop();
    }
{'fraction_non_alphanumeric': 0.07978102844835322, 'fraction_numerical': 0.005025576595171857, 'mean_word_length': 3.250190694126621, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2274172', 'n_tokens_mistral': 3555, 'n_tokens_neox': 3312, 'n_words': 680}