<commit_before>/*
 * FeatureFunctions.cpp
 *
 *  Created on: 27 Oct 2015
 *      Author: hieu
 */

#include <boost/foreach.hpp>
#include "FeatureFunctions.h"
#include "StatefulFeatureFunction.h"
#include "../System.h"
#include "../Scores.h"
#include "../MemPool.h"

#include "../TranslationModel/PhraseTable.h"
#include "../TranslationModel/UnknownWordPenalty.h"
#include "../SCFG/TargetPhraseImpl.h"
#include "../SCFG/Word.h"
#include "../PhraseBased/TargetPhraseImpl.h"
#include "util/exception.hh"

using namespace std;

namespace Moses2
{
FeatureFunctions::FeatureFunctions(System &system) :
    m_system(system), m_ffStartInd(0)
{
  //m_registry.PrintFF();
}

FeatureFunctions::~FeatureFunctions()
{
  RemoveAllInColl(m_featureFunctions);
}

void FeatureFunctions::Load()
{
  // load, everything but pts
  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
  FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
  PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);

  if (pt) {
    // do nothing. load pt last
  }
  else {
    cerr << "Loading " << nonConstFF->GetName() << endl;
    nonConstFF->Load(m_system);
    cerr << "Finished loading " << nonConstFF->GetName() << endl;
  }
}

// load pt
BOOST_FOREACH(const PhraseTable *pt, m_phraseTables) {
  PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
  cerr << "Loading " << nonConstPT->GetName() << endl;
  nonConstPT->Load(m_system);
  cerr << "Finished loading " << nonConstPT->GetName() << endl;
}
}

void FeatureFunctions::Create()
{
  const Parameter &params = m_system.params;

  const PARAM_VEC *ffParams = params.GetParam("feature");
  UTIL_THROW_IF2(ffParams == NULL, "Must have [feature] section");

  BOOST_FOREACH(const std::string &line, *ffParams){
  //cerr << "line=" << line << endl;
  FeatureFunction *ff = Create(line);

  m_featureFunctions.push_back(ff);

  StatefulFeatureFunction *sfff = dynamic_cast<StatefulFeatureFunction*>(ff);
  if (sfff) {
    sfff->SetStatefulInd(m_statefulFeatureFunctions.size());
    m_statefulFeatureFunctions.push_back(sfff);
  }

  if (ff->HasPhraseTableInd()) {
    ff->SetPhraseTableInd(m_withPhraseTableInd.size());
    m_withPhraseTableInd.push_back(ff);
  }

  PhraseTable *pt = dynamic_cast<PhraseTable*>(ff);
  if (pt) {
    pt->SetPtInd(m_phraseTables.size());
    m_phraseTables.push_back(pt);
  }

  const UnknownWordPenalty *unkWP = dynamic_cast<const UnknownWordPenalty *>(pt);
  if (unkWP) {
	  m_unkWP = unkWP;
  }

}
}

FeatureFunction *FeatureFunctions::Create(const std::string &line)
{
  vector<string> toks = Tokenize(line);

  FeatureFunction *ff = m_registry.Construct(m_ffStartInd, toks[0], line);
  UTIL_THROW_IF2(ff == NULL, "Feature function not created");

  // name
  if (ff->GetName() == "") {
    ff->SetName(GetDefaultName(toks[0]));
  }<commit_msg>implement –mark-unknown, -unknown-word-prefix, and –unknown-word-suffix for backward compatibility<commit_after>/*
 * FeatureFunctions.cpp
 *
 *  Created on: 27 Oct 2015
 *      Author: hieu
 */

#include <boost/foreach.hpp>
#include "FeatureFunctions.h"
#include "StatefulFeatureFunction.h"
#include "../System.h"
#include "../Scores.h"
#include "../MemPool.h"

#include "../TranslationModel/PhraseTable.h"
#include "../TranslationModel/UnknownWordPenalty.h"
#include "../SCFG/TargetPhraseImpl.h"
#include "../SCFG/Word.h"
#include "../PhraseBased/TargetPhraseImpl.h"
#include "util/exception.hh"

using namespace std;

namespace Moses2
{
FeatureFunctions::FeatureFunctions(System &system) :
    m_system(system), m_ffStartInd(0)
{
  //m_registry.PrintFF();
}

FeatureFunctions::~FeatureFunctions()
{
  RemoveAllInColl(m_featureFunctions);
}

void FeatureFunctions::Load()
{
  // load, everything but pts
  BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
  FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
  PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);

  if (pt) {
    // do nothing. load pt last
  }
  else {
    cerr << "Loading " << nonConstFF->GetName() << endl;
    nonConstFF->Load(m_system);
    cerr << "Finished loading " << nonConstFF->GetName() << endl;
  }
}

// load pt
BOOST_FOREACH(const PhraseTable *pt, m_phraseTables) {
  PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
  cerr << "Loading " << nonConstPT->GetName() << endl;
  nonConstPT->Load(m_system);
  cerr << "Finished loading " << nonConstPT->GetName() << endl;
}
}

void FeatureFunctions::Create()
{
  const Parameter &params = m_system.params;

  const PARAM_VEC *ffParams = params.GetParam("feature");
  UTIL_THROW_IF2(ffParams == NULL, "Must have [feature] section");

  BOOST_FOREACH(const std::string &line, *ffParams){
    //cerr << "line=" << line << endl;
    FeatureFunction *ff = Create(line);

    m_featureFunctions.push_back(ff);

    StatefulFeatureFunction *sfff = dynamic_cast<StatefulFeatureFunction*>(ff);
    if (sfff) {
      sfff->SetStatefulInd(m_statefulFeatureFunctions.size());
      m_statefulFeatureFunctions.push_back(sfff);
    }

    if (ff->HasPhraseTableInd()) {
      ff->SetPhraseTableInd(m_withPhraseTableInd.size());
      m_withPhraseTableInd.push_back(ff);
    }

    PhraseTable *pt = dynamic_cast<PhraseTable*>(ff);
    if (pt) {
      pt->SetPtInd(m_phraseTables.size());
      m_phraseTables.push_back(pt);
    }

    UnknownWordPenalty *unkWP = dynamic_cast<UnknownWordPenalty *>(pt);
    if (unkWP) {
      m_unkWP = unkWP;

      // legacy support
      if (m_system.options.unk.drop) {
        unkWP->SetParameter("drop", "true");
      }
      if (m_system.options.unk.mark) {
        unkWP->SetParameter("prefix", m_system.options.unk.prefix);
        unkWP->SetParameter("suffix", m_system.options.unk.suffix);
      }
    }

  }
}

FeatureFunction *FeatureFunctions::Create(const std::string &line)
{
  vector<string> toks = Tokenize(line);

  FeatureFunction *ff = m_registry.Construct(m_ffStartInd, toks[0], line);
  UTIL_THROW_IF2(ff == NULL, "Feature function not created");

  // name
  if (ff->GetName() == "") {
    ff->SetName(GetDefaultName(toks[0]));
  }
{'fraction_non_alphanumeric': 0.15397296406198482, 'fraction_numerical': 0.003956478733926805, 'mean_word_length': 4.407308377896613, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5613774', 'n_tokens_mistral': 2141, 'n_tokens_neox': 2050, 'n_words': 369}