<commit_before>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electronic Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */

package edu.cmu.sphinx.jsapi;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

import javax.speech.EngineException;
import javax.speech.recognition.GrammarException;
import javax.speech.recognition.GrammarSyntaxDetail;
import javax.speech.recognition.Recognizer;
import javax.speech.recognition.Rule;
import javax.speech.recognition.RuleAlternatives;
import javax.speech.recognition.RuleCount;
import javax.speech.recognition.RuleGrammar;
import javax.speech.recognition.RuleName;
import javax.speech.recognition.RuleParse;
import javax.speech.recognition.RuleSequence;
import javax.speech.recognition.RuleTag;
import javax.speech.recognition.RuleToken;

import com.sun.speech.engine.recognition.BaseRecognizer;

import edu.cmu.sphinx.linguist.language.grammar.Grammar;
import edu.cmu.sphinx.linguist.language.grammar.GrammarNode;
import edu.cmu.sphinx.util.LogMath;
import edu.cmu.sphinx.util.props.PropertyException;
import edu.cmu.sphinx.util.props.PropertySheet;
import edu.cmu.sphinx.util.props.PropertyType;
import edu.cmu.sphinx.util.props.Registry;

/**
 * Defines a BNF-style grammar based on JSGF grammar rules in a file.
 * 
 * <p>
 * The Java Speech Grammar Format (JSGF) is a BNF-style, platform-independent,
 * and vendor-independent textual representation of grammars for use in speech
 * recognition. It is used by the <a
 * href="http://java.sun.com/products/java-media/speech/">Java Speech API
 * (JSAPI) </a>.
 * 
 * <p>
 * Here we only intend to give a couple of examples of grammars written in
 * JSGF, so that you can quickly learn to write your own grammars. For more
 * examples and a complete specification of JSGF, go to
 * <p><a
 * href="http://java.sun.com/products/java-media/speech/forDevelopers/JSGF/">http://java.sun.com/products/java-media/speech/forDevelopers/JSGF/
 * </a>.
 * 
 * <p>
 * <b>Example 1: "Hello World" in JSGF </b>
 * 
 * <p>
 * The example below shows how a JSGF grammar that generates the sentences
 * "Hello World":
 * 
 * <p>
 * <table width="100%" cellpadding="10">
 * <tr>
 * <td bgcolor="#DDDDDD">
 * 
 * <pre>
 *  #JSGF V1.0
 *  
 *  public &lt;helloWorld&gt; = Hello World;
 * </pre>
 * 
 * </td>
 * </tr>
 * </table>
 * 
 * <i>Figure 1: Hello grammar that generates the sentences "Hello World". </i>
 * 
 * <p>
 * The above grammar is saved in a file called "hello.gram". It defines a
 * public grammar rule called "helloWorld". In order for this grammar rule to
 * be publicly accessible, we must be declared it "public". Non-public grammar
 * rules are not visible outside of the grammar file.
 * 
 * <p>
 * The location of the grammar file(s) is(are) defined by the
 * {@link #PROP_BASE_GRAMMAR_URL baseGrammarURL}property. Since all JSGF
 * grammar files end with ".gram", it will automatically search all such files
 * at the given URL for the grammar. The name of the grammar to search for is
 * specified by {@link #PROP_GRAMMAR_NAME grammarName}. In this example, the
 * grammar name is "helloWorld".
 * 
 * <p>
 * <b>Example 2: Command Grammar in JSGF </b>
 * 
 * <p>
 * This examples shows a grammar that generates basic control commands like
 * "move a menu thanks please", "close file", "oh mighty computer please kindly
 * delete menu thanks". It is the same as one of the command & control examples
 * in the <a
 * href="http://java.sun.com/products/java-media/speech/forDevelopers/JSGF/">JSGF
 * specification </a>. It is considerably more complex than the previous
 * example. It defines the public grammar called "basicCmd".
 * 
 * <p>
 * <table width="100%" cellpadding="10">
 * <tr>
 * <td bgcolor="#DDDDDD">
 * 
 * <pre>
 *  #JSGF V1.0
 *  
 *  public &lt;basicCmd&gt; = &lt;startPolite&gt; &lt;command&gt; &lt;endPolite&gt;;
 *  
 *  &lt;command&gt; = &lt;action&gt; &lt;object&gt;;
 *  &lt;action&gt; = /10/ open |/2/ close |/1/ delete |/1/ move;
 *  &lt;object&gt; = [the | a] (window | file | menu);
 *  
 *  &lt;startPolite&gt; = (please | kindly | could you | oh mighty computer) *;
 *  &lt;endPolite&gt; = [ please | thanks | thank you ];
 * </pre>
 * 
 * </td>
 * </tr>
 * </table>
 * 
 * <i>Figure 2: Command grammar that generates simple control commands. </i>
 * 
 * <p>
 * The features of JSGF that are shown in this example includes:
 * <ul>
 * <li>using other grammar rules within a grammar rule.
 * <li>the OR "|" operator.
 * <li>the grouping "(...)" operator.
 * <li>the optional grouping "[...]" operator.
 * <li>the zero-or-many "*" (called Kleene star) operator.
 * <li>a probability (e.g., "open" is more likely than the others).
 * </ul>
 * 
 * <p>
 * <h3>From JSGF to Grammar Graph</h3>
 * 
 * After the JSGF grammar is read in, it is converted to a graph of words
 * representing the grammar. Lets call this the grammar graph. It is from this
 * grammar graph that the eventual search structure used for speech recognition
 * is built. Below, we show the grammar graphs created from the above JSGF
 * grammars. The nodes <code>"&lt;sil&gt;"</code> means "silence".
 * 
 * <p>
 * <img src="doc-files/helloWorld.jpg"> <br>
 * <i>Figure 3: Grammar graph created from the Hello World grammar. </i>
 * 
 * <p>
 * <img src="doc-files/commandGrammar.jpg"> <br>
 * <i>Figure 4: Grammar graph created from the Command grammar. </i>
 * 
 * <p>
 * <h3>Implementation Notes</h3>
 * <ol>
 * <li>This implementation does not support right-hand recursion.
 * <li>All probabilities are maintained in LogMath log base.
 * </ol>
 */
public class JSGFGrammar extends Grammar {

    /**
     * Sphinx property that defines the location of the JSGF grammar file.
     */
    public final static String PROP_BASE_GRAMMAR_URL = "baseGrammarURL";

    /**
     * Default value for the location of the JSGF grammar file.
     */
    public final static String PROP_BASE_GRAMMAR_URL_DEFAULT = "file:./";

    /**
     * Sphinx property that defines the location of the JSGF grammar file.
     */
    public final static String PROP_GRAMMAR_NAME = "grammarName";

    /**
     * Default value for PROP_GRAMMAR_NAME
     */
    public final static String PROP_GRAMMAR_NAME_DEFAULT = "default.gram";
    
    /**
     * Sphinx property that defines the logMath component. 
     */
    
    public final static String PROP_LOG_MATH = "logMath";
    

    // ---------------------
    // Configurable data
    // ---------------------
    private RuleGrammar ruleGrammar;
    private int identity;
    private Map ruleNameStack = new HashMap();
    private Recognizer recognizer;
    private String urlString;
    private String grammarName;
    private URL baseURL = null;
    private LogMath logMath;

    /*
     * (non-Javadoc)
     * 
     * @see edu.cmu.sphinx.util.props.Configurable#register(java.lang.String,
     *      edu.cmu.sphinx.util.props.Registry)
     */
    public void register(String name, Registry registry)
            throws PropertyException {
        super.register(name, registry);
        registry.register(PROP_BASE_GRAMMAR_URL, PropertyType.STRING);
        registry.register(PROP_GRAMMAR_NAME, PropertyType.STRING);
        registry.register(PROP_LOG_MATH, PropertyType.COMPONENT);
    }

    /*
     * (non-Javadoc)
     * 
     * @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
     */
    public void newProperties(PropertySheet ps) throws PropertyException {
        super.newProperties(ps);
        urlString = ps.getString(PROP_BASE_GRAMMAR_URL,
                PROP_BASE_GRAMMAR_URL_DEFAULT);
        grammarName = ps
                .getString(PROP_GRAMMAR_NAME, PROP_GRAMMAR_NAME_DEFAULT);
        logMath = (LogMath) ps.getComponent(PROP_LOG_MATH, LogMath.class);

    }

    /**
     * Returns the RuleGrammar of this JSGFGrammar.
     * 
     * @return the RuleGrammar
     */
    public RuleGrammar getRuleGrammar() {
        return ruleGrammar;
    }

    /**
     * Sets the URL context of the JSGF grammars.
     *
     * @param url the URL context of the grammars
     */
    public void setBaseURL(URL url) {
        baseURL = url;
    }

    /**
     * Creates the grammar.
     * 
     * @return the initial node of the Grammar
     */
    protected GrammarNode createGrammar() throws IOException {
        identity = 0;

        recognizer = new BaseRecognizer();

        try {
            if (baseURL == null) {
                baseURL = new URL(urlString);
            }
            recognizer.allocate();
            ruleGrammar = recognizer.loadJSGF(baseURL, grammarName);
            recognizer.commitChanges();
            ruleGrammar.setEnabled(true);

            GrammarNode firstNode = createGrammarNode(identity++, "<sil>");
            GrammarNode finalNode = createGrammarNode(identity++, "<sil>");
            finalNode.setFinalNode(true);

            // go through each rule and create a network of GrammarNodes
            // for each of them
            String[] ruleNames = ruleGrammar.listRuleNames();
            for (int i = 0; i < ruleNames.length; i++) {
                String ruleName = ruleNames[i];
                if (ruleGrammar.isRulePublic(ruleName)) {
                    debugPrintln("New Rule: " + ruleName);
                    Rule rule = ruleGrammar.getRule(ruleName);
                    GrammarGraph graph = parseRule(rule);
                    firstNode.add(graph.getStartNode(), 0.0f);
                    graph.getEndNode().add(finalNode, 0.0f);
                }
            }
            return firstNode;
        } catch (EngineException ee) {
            // ee.printStackTrace();
            throw new IOException(ee.toString());
        } catch (GrammarException ge) {
            // ge.printStackTrace();
            dumpGrammarException(ge);
            throw new IOException("GrammarException: " + ge);
        } catch (MalformedURLException mue) {
            throw new IOException("bad base grammar url " + urlString + " "
                    + mue);

        }
    }

    /**
     * Parses the given Rule into a network of GrammarNodes.<commit_msg>Changed the property 'baseGrammarURL' to 'grammarLocation', such that it can be either a URL, or a resource location.<commit_after>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electronic Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */

package edu.cmu.sphinx.jsapi;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

import javax.speech.EngineException;
import javax.speech.recognition.GrammarException;
import javax.speech.recognition.GrammarSyntaxDetail;
import javax.speech.recognition.Recognizer;
import javax.speech.recognition.Rule;
import javax.speech.recognition.RuleAlternatives;
import javax.speech.recognition.RuleCount;
import javax.speech.recognition.RuleGrammar;
import javax.speech.recognition.RuleName;
import javax.speech.recognition.RuleParse;
import javax.speech.recognition.RuleSequence;
import javax.speech.recognition.RuleTag;
import javax.speech.recognition.RuleToken;

import com.sun.speech.engine.recognition.BaseRecognizer;

import edu.cmu.sphinx.linguist.language.grammar.Grammar;
import edu.cmu.sphinx.linguist.language.grammar.GrammarNode;
import edu.cmu.sphinx.util.LogMath;
import edu.cmu.sphinx.util.props.PropertyException;
import edu.cmu.sphinx.util.props.PropertySheet;
import edu.cmu.sphinx.util.props.PropertyType;
import edu.cmu.sphinx.util.props.Registry;

/**
 * Defines a BNF-style grammar based on JSGF grammar rules in a file.
 * 
 * <p>
 * The Java Speech Grammar Format (JSGF) is a BNF-style, platform-independent,
 * and vendor-independent textual representation of grammars for use in speech
 * recognition. It is used by the <a
 * href="http://java.sun.com/products/java-media/speech/">Java Speech API
 * (JSAPI) </a>.
 * 
 * <p>
 * Here we only intend to give a couple of examples of grammars written in
 * JSGF, so that you can quickly learn to write your own grammars. For more
 * examples and a complete specification of JSGF, go to
 * <p><a
 * href="http://java.sun.com/products/java-media/speech/forDevelopers/JSGF/">http://java.sun.com/products/java-media/speech/forDevelopers/JSGF/
 * </a>.
 * 
 * <p>
 * <b>Example 1: "Hello World" in JSGF </b>
 * 
 * <p>
 * The example below shows how a JSGF grammar that generates the sentences
 * "Hello World":
 * 
 * <p>
 * <table width="100%" cellpadding="10">
 * <tr>
 * <td bgcolor="#DDDDDD">
 * 
 * <pre>
 *  #JSGF V1.0
 *  
 *  public &lt;helloWorld&gt; = Hello World;
 * </pre>
 * 
 * </td>
 * </tr>
 * </table>
 * 
 * <i>Figure 1: Hello grammar that generates the sentences "Hello World". </i>
 * 
 * <p>
 * The above grammar is saved in a file called "hello.gram". It defines a
 * public grammar rule called "helloWorld". In order for this grammar rule to
 * be publicly accessible, we must be declared it "public". Non-public grammar
 * rules are not visible outside of the grammar file.
 * 
 * <p>
 * The location of the grammar file(s) is(are) defined by the
 * {@link #PROP_GRAMMAR_LOCATION grammarLocation}property. Since all JSGF
 * grammar files end with ".gram", it will automatically search all such files
 * at the given URL for the grammar. The name of the grammar to search for is
 * specified by {@link #PROP_GRAMMAR_NAME grammarName}. In this example, the
 * grammar name is "helloWorld".
 * 
 * <p>
 * <b>Example 2: Command Grammar in JSGF </b>
 * 
 * <p>
 * This examples shows a grammar that generates basic control commands like
 * "move a menu thanks please", "close file", "oh mighty computer please kindly
 * delete menu thanks". It is the same as one of the command & control examples
 * in the <a
 * href="http://java.sun.com/products/java-media/speech/forDevelopers/JSGF/">JSGF
 * specification </a>. It is considerably more complex than the previous
 * example. It defines the public grammar called "basicCmd".
 * 
 * <p>
 * <table width="100%" cellpadding="10">
 * <tr>
 * <td bgcolor="#DDDDDD">
 * 
 * <pre>
 *  #JSGF V1.0
 *  
 *  public &lt;basicCmd&gt; = &lt;startPolite&gt; &lt;command&gt; &lt;endPolite&gt;;
 *  
 *  &lt;command&gt; = &lt;action&gt; &lt;object&gt;;
 *  &lt;action&gt; = /10/ open |/2/ close |/1/ delete |/1/ move;
 *  &lt;object&gt; = [the | a] (window | file | menu);
 *  
 *  &lt;startPolite&gt; = (please | kindly | could you | oh mighty computer) *;
 *  &lt;endPolite&gt; = [ please | thanks | thank you ];
 * </pre>
 * 
 * </td>
 * </tr>
 * </table>
 * 
 * <i>Figure 2: Command grammar that generates simple control commands. </i>
 * 
 * <p>
 * The features of JSGF that are shown in this example includes:
 * <ul>
 * <li>using other grammar rules within a grammar rule.
 * <li>the OR "|" operator.
 * <li>the grouping "(...)" operator.
 * <li>the optional grouping "[...]" operator.
 * <li>the zero-or-many "*" (called Kleene star) operator.
 * <li>a probability (e.g., "open" is more likely than the others).
 * </ul>
 * 
 * <p>
 * <h3>From JSGF to Grammar Graph</h3>
 * 
 * After the JSGF grammar is read in, it is converted to a graph of words
 * representing the grammar. Lets call this the grammar graph. It is from this
 * grammar graph that the eventual search structure used for speech recognition
 * is built. Below, we show the grammar graphs created from the above JSGF
 * grammars. The nodes <code>"&lt;sil&gt;"</code> means "silence".
 * 
 * <p>
 * <img src="doc-files/helloWorld.jpg"> <br>
 * <i>Figure 3: Grammar graph created from the Hello World grammar. </i>
 * 
 * <p>
 * <img src="doc-files/commandGrammar.jpg"> <br>
 * <i>Figure 4: Grammar graph created from the Command grammar. </i>
 * 
 * <p>
 * <h3>Implementation Notes</h3>
 * <ol>
 * <li>This implementation does not support right-hand recursion.
 * <li>All probabilities are maintained in LogMath log base.
 * </ol>
 */
public class JSGFGrammar extends Grammar {

    /**
     * Sphinx property that defines the location of the JSGF grammar file.
     * This property can either be the URL of the grammar location
     * (e.g., "file:./" for the current directory), or a resource locator
     * (e.g., "/demo/sphinx/helloworld", if an application is deployed 
     * within a JAR file, and the grammar files reside in the directory
     * "/demo/sphinx/helloworld" within the JAR file.) In the case of
     * a resource locator, the method <code>Class.getResource(grammarLocation)
     * </code> will be used to obtain the URL of that location.
     * In either case, we eventually get a URL, which is then passed
     * into the JSAPI method <code>loadJSGF(URL context, String grammarName)
     * </code>.
     */
    public final static String PROP_GRAMMAR_LOCATION = "grammarLocation";

    /**
     * Default value for the location of the JSGF grammar file.
     */
    public final static String PROP_GRAMMAR_LOCATION_DEFAULT = "file:./";

    /**
     * Sphinx property that defines the location of the JSGF grammar file.
     */
    public final static String PROP_GRAMMAR_NAME = "grammarName";

    /**
     * Default value for PROP_GRAMMAR_NAME
     */
    public final static String PROP_GRAMMAR_NAME_DEFAULT = "default.gram";
    
    /**
     * Sphinx property that defines the logMath component. 
     */
    public final static String PROP_LOG_MATH = "logMath";
    

    // ---------------------
    // Configurable data
    // ---------------------
    private RuleGrammar ruleGrammar;
    private int identity;
    private Map ruleNameStack = new HashMap();
    private Recognizer recognizer;
    private String location;
    private String grammarName;
    private URL baseURL = null;
    private LogMath logMath;

    /*
     * (non-Javadoc)
     * 
     * @see edu.cmu.sphinx.util.props.Configurable#register(java.lang.String,
     *      edu.cmu.sphinx.util.props.Registry)
     */
    public void register(String name, Registry registry)
            throws PropertyException {
        super.register(name, registry);
        registry.register(PROP_GRAMMAR_LOCATION, PropertyType.STRING);
        registry.register(PROP_GRAMMAR_NAME, PropertyType.STRING);
        registry.register(PROP_LOG_MATH, PropertyType.COMPONENT);
    }

    /*
     * (non-Javadoc)
     * 
     * @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
     */
    public void newProperties(PropertySheet ps) throws PropertyException {
        super.newProperties(ps);
        location = ps.getString(PROP_GRAMMAR_LOCATION,
                                PROP_GRAMMAR_LOCATION_DEFAULT);
        grammarName = ps.getString(PROP_GRAMMAR_NAME,
                                   PROP_GRAMMAR_NAME_DEFAULT);
        logMath = (LogMath) ps.getComponent(PROP_LOG_MATH, LogMath.class);

    }

    /**
     * Returns the RuleGrammar of this JSGFGrammar.
     * 
     * @return the RuleGrammar
     */
    public RuleGrammar getRuleGrammar() {
        return ruleGrammar;
    }

    /**
     * Returns true if the given string is a well-formed URL.
     *
     * @param url the URL string to test
     *
     * @return true if the given string is a well-formed URL
     */
    private boolean isURL(String url) {
        try {
            URL u = new URL(url);
            return true;
        } catch (MalformedURLException me) {
            return false;
        }
    }

    /**
     * Creates the grammar.
     * 
     * @return the initial node of the Grammar
     */
    protected GrammarNode createGrammar() throws IOException {
        identity = 0;

        recognizer = new BaseRecognizer();

        try {
            if (isURL(location)) {
                baseURL = new URL(location);
            } else {
                baseURL = this.getClass().getResource(location);
            }

            recognizer.allocate();
            ruleGrammar = recognizer.loadJSGF(baseURL, grammarName);
            recognizer.commitChanges();
            ruleGrammar.setEnabled(true);

            GrammarNode firstNode = createGrammarNode(identity++, "<sil>");
            GrammarNode finalNode = createGrammarNode(identity++, "<sil>");
            finalNode.setFinalNode(true);

            // go through each rule and create a network of GrammarNodes
            // for each of them
            String[] ruleNames = ruleGrammar.listRuleNames();
            for (int i = 0; i < ruleNames.length; i++) {
                String ruleName = ruleNames[i];
                if (ruleGrammar.isRulePublic(ruleName)) {
                    debugPrintln("New Rule: " + ruleName);
                    Rule rule = ruleGrammar.getRule(ruleName);
                    GrammarGraph graph = parseRule(rule);
                    firstNode.add(graph.getStartNode(), 0.0f);
                    graph.getEndNode().add(finalNode, 0.0f);
                }
            }
            return firstNode;
        } catch (EngineException ee) {
            // ee.printStackTrace();
            throw new IOException(ee.toString());
        } catch (GrammarException ge) {
            // ge.printStackTrace();
            dumpGrammarException(ge);
            throw new IOException("GrammarException: " + ge);
        } catch (MalformedURLException mue) {
            throw new IOException("bad base grammar url " + location + 
                                  " " + mue);
        }
    }

    /**
     * Parses the given Rule into a network of GrammarNodes.
{'fraction_non_alphanumeric': 0.11535668905427482, 'fraction_numerical': 0.004628789253948085, 'mean_word_length': 2.776045236463331, 'pattern_counts': {'":': 2, '<': 157, '<?xml version=': 0, '>': 155, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6597072', 'n_tokens_mistral': 7011, 'n_tokens_neox': 6549, 'n_words': 2003}