<commit_before>#pragma once

#include <string>
#include "VWFeatureSource.h"

namespace Moses
{
  
class VWFeatureBagOfWords : public VWFeatureSource
{
  public:
    VWFeatureBagOfWords(const std::string &line)
      : VWFeatureSource(line)
    {}
  
    void operator()(const InputType &input
                  , const InputPath &inputPath
                  , const WordsRange &sourceRange
                  , Discriminative::Classifier *classifier) const
    {
      std::cerr << GetScoreProducerDescription() << " got Phrase: " << sourceRange << std::endl;
    }
};

}<commit_msg>bow feature<commit_after>#pragma once

#include <string>
#include "VWFeatureSource.h"

namespace Moses
{
  
class VWFeatureBagOfWords : public VWFeatureSource
{
  public:
    VWFeatureBagOfWords(const std::string &line)
      : VWFeatureSource(line)
    {
      ReadParameters();
    }
  
    void operator()(const InputType &input
                  , const InputPath &inputPath
                  , const WordsRange &sourceRange
                  , Discriminative::Classifier *classifier) const
    {
      for (size_t i = 0; i < input.GetSize(); i++) {
        classifier->AddLabelIndependentFeature("bow^" + input.GetWord(i).GetString(m_sourceFactors, false));
      }
    }
};

}
{'fraction_non_alphanumeric': 0.10759493670886076, 'fraction_numerical': 0.0007911392405063291, 'mean_word_length': 2.7537091988130564, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5810876', 'n_tokens_mistral': 394, 'n_tokens_neox': 377, 'n_words': 78}