<commit_before>#include "PT.h"

int PT::s_index = 0;

PT::PT(const std::string &line, int numFeatures, bool isHierarchical)
:FF(line)
{
  index = s_index++;
  this->numFeatures = numFeatures;    
  path = toks[0];

  inFactors.push_back(0);
  outFactors.push_back(0);

  int implementation;
  if (toks.size() > 1)
    implementation = Scan<int>(toks[1]);
  else if (isHierarchical)
    implementation = 6;
  else
    implementation = 0;

  switch (implementation)
  {
  case 0: name = "PhraseDictionaryMemory"; break;
  case 1: name = "PhraseDictionaryTreeAdaptor"; break;
  case 2: name = "PhraseDictionaryOnDisk"; break;
  case 6: name = "PhraseDictionarySCFG"; break;
  default:name = "UnknownPtImplementation"; break;
  }
}

void PT::Output(std::ostream &out) const
{
  out << name << " "
      << " num-features=" << numFeatures
      << " path=" << path;

  out << " input-factor=" << OutputFactors(inFactors);
  out << " output-factor=" << OutputFactors(outFactors);

  out << std::endl;
}

<commit_msg>delete space<commit_after>#include "PT.h"

int PT::s_index = 0;

PT::PT(const std::string &line, int numFeatures, bool isHierarchical)
:FF(line)
{
  index = s_index++;
  this->numFeatures = numFeatures;    
  path = toks[0];

  inFactors.push_back(0);
  outFactors.push_back(0);

  int implementation;
  if (toks.size() > 1)
    implementation = Scan<int>(toks[1]);
  else if (isHierarchical)
    implementation = 6;
  else
    implementation = 0;

  switch (implementation)
  {
  case 0: name = "PhraseDictionaryMemory"; break;
  case 1: name = "PhraseDictionaryTreeAdaptor"; break;
  case 2: name = "PhraseDictionaryOnDisk"; break;
  case 6: name = "PhraseDictionarySCFG"; break;
  default:name = "UnknownPtImplementation"; break;
  }
}

void PT::Output(std::ostream &out) const
{
  out << name
      << " num-features=" << numFeatures
      << " path=" << path;

  out << " input-factor=" << OutputFactors(inFactors);
  out << " output-factor=" << OutputFactors(outFactors);

  out << std::endl;
}


{'fraction_non_alphanumeric': 0.1550695825049702, 'fraction_numerical': 0.011928429423459244, 'mean_word_length': 3.9950372208436726, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6474339', 'n_tokens_mistral': 741, 'n_tokens_neox': 699, 'n_words': 159}