<commit_before>/**
 * @file analyzer.cpp
 */

#include "analyzers/all.h"
#include "analyzers/token_stream.h"
#include "analyzers/filters/alpha_filter.h"
#include "analyzers/filters/english_normalizer.h"
#include "analyzers/filters/sentence_boundary.h"
#include "analyzers/filters/length_filter.h"
#include "analyzers/filters/list_filter.h"
#include "analyzers/filters/lowercase_filter.h"
#include "analyzers/filters/porter2_stemmer.h"
#include "analyzers/tokenizers/whitespace_tokenizer.h"
#include "analyzers/tokenizers/character_tokenizer.h"
#include "cpptoml.h"
#include "corpus/document.h"
#include "io/mmap_file.h"
#include "stemmers/no_stemmer.h"
#include "stemmers/porter2.h"
#include "util/shim.h"

namespace meta {
namespace analyzers {

std::string analyzer::get_content(const corpus::document & doc)
{
    if(doc.contains_content())
        return doc.content();

    io::mmap_file file{doc.path()};
    return {file.begin(), file.size()};
}

io::parser analyzer::create_parser(const corpus::document & doc,
        const std::string & extension, const std::string & delims)
{
    if(doc.contains_content())
        return io::parser{doc.content(), delims,
                          io::parser::input_type::String};
    else
        return io::parser{doc.path() + extension, delims,
                          io::parser::input_type::File};
}

std::unique_ptr<token_stream>
    analyzer::default_filter_chain(const cpptoml::toml_group& config)
{

    auto stopwords = config.get_as<std::string>("stop-words");

    std::unique_ptr<token_stream> result;

    result = make_unique<whitespace_tokenizer>();
    result = make_unique<english_normalizer>(std::move(result));
    sentence_boundary::load_heuristics(config);
    result = make_unique<sentence_boundary>(std::move(result));
    result = make_unique<alpha_filter>(std::move(result));
    result = make_unique<length_filter>(std::move(result), 2, 35);
    result = make_unique<lowercase_filter>(std::move(result));
    result = make_unique<list_filter>(std::move(result), *stopwords);
    result = make_unique<porter2_stemmer>(std::move(result));<commit_msg>Change ordering of default filter chain.<commit_after>/**
 * @file analyzer.cpp
 */

#include "analyzers/all.h"
#include "analyzers/token_stream.h"
#include "analyzers/filters/alpha_filter.h"
#include "analyzers/filters/english_normalizer.h"
#include "analyzers/filters/sentence_boundary.h"
#include "analyzers/filters/length_filter.h"
#include "analyzers/filters/list_filter.h"
#include "analyzers/filters/lowercase_filter.h"
#include "analyzers/filters/porter2_stemmer.h"
#include "analyzers/tokenizers/whitespace_tokenizer.h"
#include "analyzers/tokenizers/character_tokenizer.h"
#include "cpptoml.h"
#include "corpus/document.h"
#include "io/mmap_file.h"
#include "stemmers/no_stemmer.h"
#include "stemmers/porter2.h"
#include "util/shim.h"

namespace meta {
namespace analyzers {

std::string analyzer::get_content(const corpus::document & doc)
{
    if(doc.contains_content())
        return doc.content();

    io::mmap_file file{doc.path()};
    return {file.begin(), file.size()};
}

io::parser analyzer::create_parser(const corpus::document & doc,
        const std::string & extension, const std::string & delims)
{
    if(doc.contains_content())
        return io::parser{doc.content(), delims,
                          io::parser::input_type::String};
    else
        return io::parser{doc.path() + extension, delims,
                          io::parser::input_type::File};
}

std::unique_ptr<token_stream>
    analyzer::default_filter_chain(const cpptoml::toml_group& config)
{

    auto stopwords = config.get_as<std::string>("stop-words");

    std::unique_ptr<token_stream> result;

    result = make_unique<whitespace_tokenizer>();
    result = make_unique<english_normalizer>(std::move(result));
    sentence_boundary::load_heuristics(config);
    result = make_unique<sentence_boundary>(std::move(result));
    result = make_unique<lowercase_filter>(std::move(result));
    result = make_unique<alpha_filter>(std::move(result));
    result = make_unique<length_filter>(std::move(result), 2, 35);
    result = make_unique<list_filter>(std::move(result), *stopwords);
    result = make_unique<porter2_stemmer>(std::move(result));
{'fraction_non_alphanumeric': 0.14767142522817692, 'fraction_numerical': 0.002808331383103206, 'mean_word_length': 5.167388167388167, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3278383', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1403, 'n_words': 210}