<commit_before>/* This file is part of the Joshua Machine Translation System.
 * 
 * Joshua is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1
 * of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 * MA 02111-1307 USA
 */

package joshua.decoder.ff.lm;

// BUG: At best we should use List, but we use int[] everywhere to
// represent phrases therefore these additional methods are excessive.
import java.util.ArrayList;

/**
 * An interface for new language models to implement. An object of
 * this type is passed to LanguageModelFF, which will handle all
 * the dynamic programming and state maintinence.
 *
 * All the function here should return LogP, not the cost.
 *
 * @author <NAME> thornton <<EMAIL>>
 * @version $LastChangedDate: 2008-07-28 18:44:45 -0400 (Mon, 28 Jul 2008) $
 */
public interface NGramLanguageModel {
	
//===============================================================
// Attributes
//===============================================================
	int getOrder();
	
//===============================================================
// Methods
//===============================================================

	// BUG: why do we pass the order? Does this method reduce the order as well?
	/**
	 * @param sentence   the sentence to be scored
	 * @param order      the order of N-grams for the LM
	 * @param startIndex the index of first event-word we want
	 *                   to get its probability; if we want to
	 *                   get the prob for the whole sentence,
	 *                   then startIndex should be 1
	 * @return the LogP of the whole sentence
	 */
	double sentenceLogProbability(ArrayList<Integer> sentence, int order, int startIndex);
	
	
	/**
	 * @param order used to temporarily reduce the order used by the model.
	 */
	double ngramLogProbability(ArrayList<Integer> ngram, int order);
	double ngramLogProbability(int[] ngram, int order);
	double ngramLogProbability(int[] ngram);
	
	
//===============================================================
// Equivalent LM State (use DefaultNGramLanguageModel if you don't care)
//===============================================================
	double logProbabilityOfBackoffState(
		ArrayList<Integer> ngram, int order, int qtyAdditionalBackoffWeight);
	double logProbabilityOfBackoffState(
		int[] ngram, int order, int qtyAdditionalBackoffWeight);
	
	int[] leftEquivalentState(int[] originalState, int order, double[] cost);
	int[] rightEquivalentState(int[] originalState, int order);
	
}<commit_msg>Added comment on what logProbabilityOfBackoffState means<commit_after>/* This file is part of the Joshua Machine Translation System.
 * 
 * Joshua is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1
 * of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 * MA 02111-1307 USA
 */

package joshua.decoder.ff.lm;

// BUG: At best we should use List, but we use int[] everywhere to
// represent phrases therefore these additional methods are excessive.
import java.util.ArrayList;

/**
 * An interface for new language models to implement. An object of
 * this type is passed to LanguageModelFF, which will handle all
 * the dynamic programming and state maintinence.
 *
 * All the function here should return LogP, not the cost.
 *
 * @author wren ng thornton <<EMAIL>>
 * @version $LastChangedDate: 2008-07-28 18:44:45 -0400 (Mon, 28 Jul 2008) $
 */
public interface NGramLanguageModel {
	
//===============================================================
// Attributes
//===============================================================
	int getOrder();
	
//===============================================================
// Methods
//===============================================================

	// BUG: why do we pass the order? Does this method reduce the order as well?
	/**
	 * @param sentence   the sentence to be scored
	 * @param order      the order of N-grams for the LM
	 * @param startIndex the index of first event-word we want
	 *                   to get its probability; if we want to
	 *                   get the prob for the whole sentence,
	 *                   then startIndex should be 1
	 * @return the LogP of the whole sentence
	 */
	double sentenceLogProbability(ArrayList<Integer> sentence, int order, int startIndex);
	
	
	/**
	 * @param order used to temporarily reduce the order used by the model.
	 */
	double ngramLogProbability(ArrayList<Integer> ngram, int order);
	double ngramLogProbability(int[] ngram, int order);
	double ngramLogProbability(int[] ngram);
	
	
//===============================================================
// Equivalent LM State (use DefaultNGramLanguageModel if you don't care)
//===============================================================
	
	/**
	 * This returns the log probability of the special backoff
	 * symbol used to fill out contexts which have been backed-off.
	 * The LanguageModelFF implementation is to call this
	 * unigram probability for each such token, and then call
	 * ngramLogProbability for the remaining actual N-gram.
	 *
	 * @todo Is this really the best interface?
	 */
	double logProbabilityOfBackoffState(
		ArrayList<Integer> ngram, int order, int qtyAdditionalBackoffWeight);
	double logProbabilityOfBackoffState(
		int[] ngram, int order, int qtyAdditionalBackoffWeight);
	
	int[] leftEquivalentState(int[] originalState, int order, double[] cost);
	int[] rightEquivalentState(int[] originalState, int order);
	
}
{'fraction_non_alphanumeric': 0.18163914558400243, 'fraction_numerical': 0.012422360248447204, 'mean_word_length': 4.169929522317933, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3742044', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1556, 'n_words': 742}