<commit_before>#include <sstream>
#include "TargetWordInsertionFeature.h"
#include "Phrase.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "ChartHypothesis.h"
#include "ScoreComponentCollection.h"
#include "TranslationOption.h"
#include "UserMessage.h"

namespace Moses {

using namespace std;

TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line)
:StatelessFeatureFunction("twi", ScoreProducer::unlimited, line),
m_unrestricted(true)
{
  std::cerr << "Initializing target word insertion feature.." << std::endl;

  vector<string> tokens = Tokenize(line);
  //CHECK(tokens[0] == m_description);

  // set factor
  m_factorType = Scan<FactorType>(tokens[1]);


  // load word list for restricted feature set
  if (tokens.size() == 3) {
    string filename = tokens[2];
    cerr << "loading target word insertion word list from " << filename << endl;
    if (!Load(filename)) {
      UserMessage::Add("Unable to load word list for target word insertion feature from file " + filename);
      //return false;
    }
  }

}

bool TargetWordInsertionFeature::Load(const std::string &filePath) 
{
  ifstream inFile(filePath.c_str());
  if (!inFile)
  {
      cerr << "could not open file " << filePath << endl;
      return false;
  }

  std::string line;
  while (getline(inFile, line)) {
    m_vocab.insert(line);
  }

  inFile.close();

  m_unrestricted = false;
  return true;<commit_msg>use key-value args for TargetWordInsertionFeature<commit_after>#include <sstream>
#include "TargetWordInsertionFeature.h"
#include "Phrase.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "ChartHypothesis.h"
#include "ScoreComponentCollection.h"
#include "TranslationOption.h"
#include "UserMessage.h"

namespace Moses {

using namespace std;

TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line)
:StatelessFeatureFunction("twi", ScoreProducer::unlimited, line),
m_unrestricted(true)
{
  std::cerr << "Initializing target word insertion feature.." << std::endl;

  string filename;

  for (size_t i = 0; i < m_args.size(); ++i) {
    const vector<string> &args = m_args[i];

    if (args[0] == "factor") {
      m_factorType = Scan<FactorType>(args[1]);
    }
    else if (args[0] == "path") {
      filename = args[1];
    }

  }

  // load word list for restricted feature set
  if (filename != "") {
    cerr << "loading target word insertion word list from " << filename << endl;
    if (!Load(filename)) {
      UserMessage::Add("Unable to load word list for target word insertion feature from file " + filename);
      //return false;
    }
  }

}

bool TargetWordInsertionFeature::Load(const std::string &filePath) 
{
  ifstream inFile(filePath.c_str());
  if (!inFile)
  {
      cerr << "could not open file " << filePath << endl;
      return false;
  }

  std::string line;
  while (getline(inFile, line)) {
    m_vocab.insert(line);
  }

  inFile.close();

  m_unrestricted = false;
  return true;
{'fraction_non_alphanumeric': 0.13651647612642906, 'fraction_numerical': 0.003026227303295225, 'mean_word_length': 4.22847100175747, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4921732', 'n_tokens_mistral': 991, 'n_tokens_neox': 946, 'n_words': 242}