<commit_before>#include "BiLM_NPLM.h"
#include "neuralLM.h"
#include "vocabulary.h"

namespace Moses {

BilingualLM_NPLM::BilingualLM_NPLM(const std::string &line)
      : BilingualLM(line),
        premultiply(true),
        factored(false),
        neuralLM_cache(1000000) {
          
          if (!NULL_overwrite) {
            NULL_string = "<null>"; //Default null value for nplm
          }
          FactorCollection& factorFactory = FactorCollection::Instance(); // To add null word.
          const Factor* NULL_factor = factorFactory.AddFactor(NULL_string);
          NULL_word.SetFactor(0, NULL_factor);
        }

float BilingualLM_NPLM::Score(std::vector<int>& source_words, std::vector<int>& target_words) const {
  source_words.reserve(source_ngrams+target_ngrams+1);
  source_words.insert( source_words.end(), target_words.begin(), target_words.end() );
  return m_neuralLM->lookup_ngram(source_words);
}

const Word& BilingualLM_NPLM::getNullWord() const {
  return NULL_word;
}

int BilingualLM_NPLM::getNeuralLMId(const Word& word, bool is_source_word) const {
  initSharedPointer();

  boost::unordered_map<const Factor*, int>::iterator it;
  const Factor* factor = word.GetFactor(word_factortype);

  it = neuralLMids.find(factor);
  //If we know the word return immediately
  if (it != neuralLMids.end()){
    return it->second;
  }
  //If we don't know the word and we aren't factored, return the word.
  if (!factored) {
      return unknown_word_id;
  } 
  //Else try to get a pos_factor
  const Factor* pos_factor = word.GetFactor(pos_factortype);
  it = neuralLMids.find(pos_factor);<commit_msg>floor score of bilingualNPLM<commit_after>#include "BiLM_NPLM.h"
#include "neuralLM.h"
#include "vocabulary.h"

namespace Moses {

BilingualLM_NPLM::BilingualLM_NPLM(const std::string &line)
      : BilingualLM(line),
        premultiply(true),
        factored(false),
        neuralLM_cache(1000000) {
          
          if (!NULL_overwrite) {
            NULL_string = "<null>"; //Default null value for nplm
          }
          FactorCollection& factorFactory = FactorCollection::Instance(); // To add null word.
          const Factor* NULL_factor = factorFactory.AddFactor(NULL_string);
          NULL_word.SetFactor(0, NULL_factor);
        }

float BilingualLM_NPLM::Score(std::vector<int>& source_words, std::vector<int>& target_words) const {
  source_words.reserve(source_ngrams+target_ngrams+1);
  source_words.insert( source_words.end(), target_words.begin(), target_words.end() );
  return FloorScore(m_neuralLM->lookup_ngram(source_words));
}

const Word& BilingualLM_NPLM::getNullWord() const {
  return NULL_word;
}

int BilingualLM_NPLM::getNeuralLMId(const Word& word, bool is_source_word) const {
  initSharedPointer();

  boost::unordered_map<const Factor*, int>::iterator it;
  const Factor* factor = word.GetFactor(word_factortype);

  it = neuralLMids.find(factor);
  //If we know the word return immediately
  if (it != neuralLMids.end()){
    return it->second;
  }
  //If we don't know the word and we aren't factored, return the word.
  if (!factored) {
      return unknown_word_id;
  } 
  //Else try to get a pos_factor
  const Factor* pos_factor = word.GetFactor(pos_factortype);
  it = neuralLMids.find(pos_factor);
{'fraction_non_alphanumeric': 0.11725190839694656, 'fraction_numerical': 0.00549618320610687, 'mean_word_length': 3.933734939759036, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5900532', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1066, 'n_words': 232}