<commit_before>/**
 * @file document.cpp
 * @author <NAME>
 */

#include <utility>
#include <sstream>
#include "corpus/document.h"
#include "corpus/corpus.h"
#include "util/common.h"
#include "parallel/parallel_for.h"
#include "cluster/similarity.h"

namespace meta {
namespace corpus {

document::document(const std::string & path,
                   doc_id d_id,
                   const class_label & label):
    _path{path},
    _d_id{d_id},
    _label{label},
    _length{0},
    _content{""},
    _contains_content{false}
{
    size_t idx = path.find_last_of("/") + 1;
    _name = path.substr(idx);

    // make sure class label doesn't contain a path (we only want the label, not
    // the entire file)
    size_t slash = _label.find_first_of("/\\");
    if(slash != std::string::npos)
        _label = _label.substr(0, slash);
}

void document::increment(const std::string & term, double amount)
{
    _counts[term] += amount;
    _length += amount;
}

std::string document::path() const
{
    return _path;<commit_msg>fix treating label as a string in debug mode<commit_after>/**
 * @file document.cpp
 * @author <NAME>
 */

#include <utility>
#include <sstream>
#include "corpus/document.h"
#include "corpus/corpus.h"
#include "util/common.h"
#include "parallel/parallel_for.h"
#include "cluster/similarity.h"

namespace meta {
namespace corpus {

document::document(const std::string & path,
                   doc_id d_id,
                   const class_label & label):
    _path{path},
    _d_id{d_id},
    _label{label},
    _length{0},
    _content{""},
    _contains_content{false}
{
    size_t idx = path.find_last_of("/") + 1;
    _name = path.substr(idx);

    // make sure class label doesn't contain a path (we only want the label, not
    // the entire file)
    std::string str_lbl = _label;
    size_t slash = str_lbl.find_first_of("/\\");
    if(slash != std::string::npos)
        _label = str_lbl.substr(0, slash);
}

void document::increment(const std::string & term, double amount)
{
    _counts[term] += amount;
    _length += amount;
}

std::string document::path() const
{
    return _path;
{'fraction_non_alphanumeric': 0.14507772020725387, 'fraction_numerical': 0.002826189354686764, 'mean_word_length': 3.1893491124260356, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2814959', 'n_tokens_mistral': 737, 'n_tokens_neox': 718, 'n_words': 176}