<commit_before>#include "ZambeziMiningTask.h"
#include <la-manager/AttrTokenizeWrapper.h>
#include <configuration-manager/ZambeziConfig.h>
#include <document-manager/DocumentManager.h>
#include <common/ResourceManager.h>
#include <glog/logging.h>
#include <fstream>

using namespace sf1r;

ZambeziMiningTask::ZambeziMiningTask(
    const ZambeziConfig& config,
    DocumentManager& documentManager,
    izenelib::ir::Zambezi::NewInvertedIndex& indexer)
    : config_(config)
    , documentManager_(documentManager)
    , indexer_(indexer)
    , startDocId_(0)
{
    if (config_.isDebug)
    {
        ofs_debug_.open((config_.indexFilePath + "debug").c_str(), ios::app);
    }
    
}

bool ZambeziMiningTask::buildDocument(docid_t docID, const Document& doc)
{
    std::vector<std::string> propNameList;
    std::vector<std::string> propValueList;
    propNameList.push_back("Title");
    propNameList.push_back("Attribute");
    propNameList.push_back("Category");
    propNameList.push_back("OriginalCategory");
    propNameList.push_back("Source");

    for (std::vector<std::string>::iterator i = propNameList.begin(); i != propNameList.end(); ++i)
    {
        std::string propValue;
        doc.getProperty(*i, propValue);
        propValueList.push_back(propValue);
    }

    std::vector<std::pair<std::string, double> > tokenScoreList;
    tokenScoreList = AttrTokenizeWrapper::get()->attr_tokenize_index(propValueList[0]
                                                                , propValueList[1]
                                                                , propValueList[2]<commit_msg>Rename zambezi debug file to "index.bin.debug".<commit_after>#include "ZambeziMiningTask.h"
#include <la-manager/AttrTokenizeWrapper.h>
#include <configuration-manager/ZambeziConfig.h>
#include <document-manager/DocumentManager.h>
#include <common/ResourceManager.h>
#include <glog/logging.h>
#include <fstream>

using namespace sf1r;

ZambeziMiningTask::ZambeziMiningTask(
    const ZambeziConfig& config,
    DocumentManager& documentManager,
    izenelib::ir::Zambezi::NewInvertedIndex& indexer)
    : config_(config)
    , documentManager_(documentManager)
    , indexer_(indexer)
    , startDocId_(0)
{
    if (config_.isDebug)
    {
        ofs_debug_.open((config_.indexFilePath + ".debug").c_str(), ios::app);
    }
    
}

bool ZambeziMiningTask::buildDocument(docid_t docID, const Document& doc)
{
    std::vector<std::string> propNameList;
    std::vector<std::string> propValueList;
    propNameList.push_back("Title");
    propNameList.push_back("Attribute");
    propNameList.push_back("Category");
    propNameList.push_back("OriginalCategory");
    propNameList.push_back("Source");

    for (std::vector<std::string>::iterator i = propNameList.begin(); i != propNameList.end(); ++i)
    {
        std::string propValue;
        doc.getProperty(*i, propValue);
        propValueList.push_back(propValue);
    }

    std::vector<std::pair<std::string, double> > tokenScoreList;
    tokenScoreList = AttrTokenizeWrapper::get()->attr_tokenize_index(propValueList[0]
                                                                , propValueList[1]
                                                                , propValueList[2]
{'fraction_non_alphanumeric': 0.12849677221026745, 'fraction_numerical': 0.003074085459575776, 'mean_word_length': 3.327127659574468, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7150262', 'n_tokens_mistral': 1004, 'n_tokens_neox': 994, 'n_words': 140}