<commit_before>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#ifndef moses_LanguageModelImplementation_h
#define moses_LanguageModelImplementation_h

#include <string>
#include <vector>
#include "Factor.h"
#include "TypeDef.h"
#include "Util.h"
#include "FeatureFunction.h"
#include "Word.h"
#include "LM/Base.h"

#ifdef WITH_THREADS
#include <boost/shared_ptr.hpp>
#endif

namespace Moses
{

class LanguageModel;

class FactorCollection;
class Factor;
class Phrase;

struct LMResult {
  // log probability
  float score;
  // Is the word unknown?  
  bool unknown;
};

//! Abstract base class which represent a language model on a contiguous phrase
class LanguageModelImplementation
{
#ifndef WITH_THREADS
protected:
  /** constructor to be called by inherited class
   */
  LanguageModelImplementation() : m_referenceCount(0) {}

private:
  // ref counting handled by boost if we have threads
  unsigned int m_referenceCount;
#else
  // default constructor is ok
#endif
  void ShiftOrPush(std::vector<const Word*> &contextFactor, const Word &word) const;

protected:
  std::string	m_filePath; //! for debugging purposes
  size_t			m_nGramOrder; //! max n-gram length contained in this LM
  Word m_sentenceStartArray, m_sentenceEndArray; //! Contains factors which represents the beging and end words for this LM.
  //! Usually <s> and </s>

public:
  virtual ~LanguageModelImplementation() {}

  //! Single or multi-factor
  virtual LMType GetLMType() const = 0;

  /* whether this LM can be used on a particular phrase.
   * Should return false if phrase size = 0 or factor types required don't exists
   */
  virtual bool Useable(const Phrase &phrase) const = 0;

  /* get score of n-gram. n-gram should not be bigger than m_nGramOrder
   * Specific implementation can return State and len data to be used in hypothesis pruning
   * \param contextFactor n-gram to be scored
   * \param state LM state.  Input and output.  state must be initialized.  If state isn't initialized, you want GetValueWithoutState.
   */
  virtual LMResult GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const;

  // Like GetValueGivenState but state may not be initialized (however it is non-NULL).
  // For example, state just came from NewState(NULL).
  virtual LMResult GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const = 0;

  //! get State for a particular n-gram.  We don't care what the score is.
  // This is here so models can implement a shortcut to GetValueAndState.
  virtual void GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

  virtual const FFState *GetNullContextState() const = 0;
  virtual const FFState *GetBeginSentenceState() const = 0;
  virtual FFState *NewState(const FFState *from = NULL) const = 0;

  virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;

  virtual FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out, const LanguageModel *feature) const;

  virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection* accumulator, const LanguageModel *feature) const;

  void updateChartScore(float *prefixScore, float *finalScore, float score, size_t wordPos) const;

  //! max n-gram order of LM
  size_t GetNGramOrder() const {
    return m_nGramOrder;<commit_msg>Remove some virtual tags<commit_after>// $Id$

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#ifndef moses_LanguageModelImplementation_h
#define moses_LanguageModelImplementation_h

#include <string>
#include <vector>
#include "Factor.h"
#include "TypeDef.h"
#include "Util.h"
#include "FeatureFunction.h"
#include "Word.h"
#include "LM/Base.h"

#ifdef WITH_THREADS
#include <boost/shared_ptr.hpp>
#endif

namespace Moses
{

class LanguageModel;

class FactorCollection;
class Factor;
class Phrase;

struct LMResult {
  // log probability
  float score;
  // Is the word unknown?  
  bool unknown;
};

//! Abstract base class which represent a language model on a contiguous phrase
class LanguageModelImplementation
{
#ifndef WITH_THREADS
protected:
  /** constructor to be called by inherited class
   */
  LanguageModelImplementation() : m_referenceCount(0) {}

private:
  // ref counting handled by boost if we have threads
  unsigned int m_referenceCount;
#else
  // default constructor is ok
#endif
  void ShiftOrPush(std::vector<const Word*> &contextFactor, const Word &word) const;

protected:
  std::string	m_filePath; //! for debugging purposes
  size_t			m_nGramOrder; //! max n-gram length contained in this LM
  Word m_sentenceStartArray, m_sentenceEndArray; //! Contains factors which represents the beging and end words for this LM.
  //! Usually <s> and </s>

public:
  virtual ~LanguageModelImplementation() {}

  //! Single or multi-factor
  virtual LMType GetLMType() const = 0;

  /* whether this LM can be used on a particular phrase.
   * Should return false if phrase size = 0 or factor types required don't exists
   */
  virtual bool Useable(const Phrase &phrase) const = 0;

  /* get score of n-gram. n-gram should not be bigger than m_nGramOrder
   * Specific implementation can return State and len data to be used in hypothesis pruning
   * \param contextFactor n-gram to be scored
   * \param state LM state.  Input and output.  state must be initialized.  If state isn't initialized, you want GetValueWithoutState.
   */
  virtual LMResult GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const;

  // Like GetValueGivenState but state may not be initialized (however it is non-NULL).
  // For example, state just came from NewState(NULL).
  virtual LMResult GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const = 0;

  //! get State for a particular n-gram.  We don't care what the score is.
  // This is here so models can implement a shortcut to GetValueAndState.
  virtual void GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

  virtual const FFState *GetNullContextState() const = 0;
  virtual const FFState *GetBeginSentenceState() const = 0;
  virtual FFState *NewState(const FFState *from = NULL) const = 0;

  void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;

  FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out, const LanguageModel *feature) const;

  FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection* accumulator, const LanguageModel *feature) const;

  void updateChartScore(float *prefixScore, float *finalScore, float score, size_t wordPos) const;

  //! max n-gram order of LM
  size_t GetNGramOrder() const {
    return m_nGramOrder;
{'fraction_non_alphanumeric': 0.10634608715755405, 'fraction_numerical': 0.005779678649867067, 'mean_word_length': 4.680892974392646, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3283616', 'n_tokens_mistral': 2469, 'n_tokens_neox': 2305, 'n_words': 1005}