<commit_before>/**
 * @file ngram_word_tokenizer.cpp
 * @author <NAME>
 */

#include "tokenizers/ngram/ngram_word_tokenizer.h"
#include "util/common.h"
#include "io/parser.h"

using namespace std;

namespace meta {
namespace tokenizers {

const std::string ngram_word_tokenizer::_delimiters
    = " \n\t!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~";

ngram_word_tokenizer::ngram_word_tokenizer(
        uint16_t n,
        stopword_t stopwords,
        std::function<void(std::string &)> stemmer)
: ngram_tokenizer{n}, _stemmer{stemmer}
{
    if(stopwords != stopword_t::None)
        init_stopwords();
}

void ngram_word_tokenizer::tokenize_document(
        corpus::document & doc,
        std::function<term_id(const std::string &)> mapping)
{
    // first, get tokens

    io::parser psr{create_parser(doc, ".sen", _delimiters)};
    std::vector<std::string> tokens;
    std::string token;
    while(psr.has_next())
    {
        token = psr.next();
        if(_stopwords.find(token) == _stopwords.end())
        {
            _stemmer(token);
            tokens.push_back(token);
        }
    }

    // second, create ngrams from them

    for(size_t i = n_value() - 1; i < tokens.size(); ++i)
    {
        std::string combined = tokens[i];
        for(size_t j = 1; j < n_value(); ++j)
            combined = tokens[i - j] + "_" + combined;

        doc.increment(mapping(combined), 1);
    }
}

void ngram_word_tokenizer::init_stopwords()
{
    auto config = cpptoml::parse_file("config.toml");
    io::parser p{*config.get_as<std::string>("stop-words"), "\n"};
    while(p.has_next())
        _stopwords.insert(p.next());
}

}
}
<commit_msg>fix stop word bug in ngram_word_tokenizer<commit_after>/**
 * @file ngram_word_tokenizer.cpp
 * @author <NAME>
 */

#include "tokenizers/ngram/ngram_word_tokenizer.h"
#include "util/common.h"
#include "io/parser.h"

using namespace std;

namespace meta {
namespace tokenizers {

const std::string ngram_word_tokenizer::_delimiters
    = " \n\t!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~";

ngram_word_tokenizer::ngram_word_tokenizer(
        uint16_t n,
        stopword_t stopwords,
        std::function<void(std::string &)> stemmer)
: ngram_tokenizer{n}, _stemmer{stemmer}
{
    if(stopwords != stopword_t::None)
        init_stopwords();
}

void ngram_word_tokenizer::tokenize_document(
        corpus::document & doc,
        std::function<term_id(const std::string &)> mapping)
{
    // first, get tokens

    io::parser psr{create_parser(doc, ".sen", _delimiters)};
    std::vector<std::string> tokens;
    std::string token;
    while(psr.has_next())
    {
        token = psr.next();
        _stemmer(token);
        if(_stopwords.find(token) == _stopwords.end())
            tokens.push_back(token);
    }

    // second, create ngrams from them

    for(size_t i = n_value() - 1; i < tokens.size(); ++i)
    {
        std::string combined = tokens[i];
        for(size_t j = 1; j < n_value(); ++j)
            combined = tokens[i - j] + "_" + combined;

        doc.increment(mapping(combined), 1);
    }
}

void ngram_word_tokenizer::init_stopwords()
{
    auto config = cpptoml::parse_file("config.toml");
    io::parser p{*config.get_as<std::string>("stop-words"), "\n"};
    while(p.has_next())
    {
        std::string word{p.next()};
        _stemmer(word);
        _stopwords.insert(word);
    }
}

}
}

{'fraction_non_alphanumeric': 0.17124105011933174, 'fraction_numerical': 0.0029832935560859188, 'mean_word_length': 3.084043848964677, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6430554', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1209, 'n_words': 200}