<commit_before>#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/IndexReader.h>
#include <ir/index_manager/index/IndexBarrelWriter.h>
#include <ir/index_manager/index/FieldIndexer.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/TermPositions.h>
#include <ir/index_manager/index/EPostingWriter.h>
#include <ir/index_manager/store/IndexInput.h>

#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>
#include <util/izene_log.h>

#include <cassert>

using namespace std;

namespace bfs = boost::filesystem;

NS_IZENELIB_IR_BEGIN

namespace indexmanager
{

void writeTermInfo(
    IndexOutput* pVocWriter,
    termid_t tid,
    const TermInfo& termInfo)
{
    pVocWriter->writeInt(tid);					///write term id
    pVocWriter->writeInt(termInfo.docFreq_);		///write df
    pVocWriter->writeInt(termInfo.ctf_);			///write ctf
    pVocWriter->writeInt(termInfo.maxTF_);     ///write maxtf
    pVocWriter->writeInt(termInfo.lastDocID_);		///write last doc id
    pVocWriter->writeInt(termInfo.skipLevel_);		///write skip level
    pVocWriter->writeLong(termInfo.skipPointer_);	///write skip list offset offset
    pVocWriter->writeLong(termInfo.docPointer_);	///write document posting offset
    pVocWriter->writeInt(termInfo.docPostingLen_);	///write document posting length (without skiplist)
    pVocWriter->writeLong(termInfo.positionPointer_);	///write position posting offset
    pVocWriter->writeInt(termInfo.positionPostingLen_);///write position posting length
}

#pragma pack(push,1)
struct Record
{
    uint8_t len;
    uint32_t tid;
    uint32_t docId;
    uint32_t offset;
};
#pragma pack(pop)

FieldIndexer::FieldIndexer(
    const char* field,
    Indexer* pIndexer
)
    :pBinlog_(NULL)
    ,field_(field)
    ,pIndexer_(pIndexer)
    ,vocFilePointer_(0)
    //,alloc_(0)
    ,f_(0)
    ,termCount_(0)
    ,iHitsMax_(0)
    ,recordCount_(0)
    ,run_num_(0)
    ,pHits_(0)
    ,pHitsMax_(0)
    ,flush_(false)
{
    skipInterval_ = pIndexer_->getSkipInterval();
    maxSkipLevel_ = pIndexer_->getMaxSkipLevel();
    indexLevel_ = pIndexer_->pConfigurationManager_->indexStrategy_.indexLevel_;

    sorterFileName_ = field_+".tmp";
    bfs::path path(bfs::path(pIndexer_->pConfigurationManager_->indexStrategy_.indexLocation_)
                   /bfs::path(sorterFileName_));
    sorterFullPath_ = path.string();
}

FieldIndexer::~FieldIndexer()
{
    if (f_)
    {
        fclose(f_);
        if(! boost::filesystem::remove(sorterFullPath_))
            LOG(WARNING) << "FieldIndexer::~FieldIndexer(): failed to remove file " << sorterFullPath_;
    }

    deletebinlog();
}

void FieldIndexer::deletebinlog()
{
    boost::filesystem::remove(BinlogPath_);
}

void FieldIndexer::checkBinlog()
{
    BarrelInfo* pCurBarrelInfo = pIndexer_->getIndexWriter()->getBarrelInfo();

    if(pBinlog_ == NULL)
    {
        pBinlog_ = new Binlog(pIndexer_);
        std::string BinlogName = field_ + ".binlog";
        bfs::path path(bfs::path(pIndexer_->pConfigurationManager_->indexStrategy_.indexLocation_)
                       /bfs::path(BinlogName));
        BinlogPath_ = path.string();
        if(pBinlog_->openForRead(BinlogPath_))
        {
            if(!pCurBarrelInfo)
                pIndexer_->getIndexWriter()->createBarrelInfo();
            if (pIndexer_->isRealTime())
            {
                if (!pIndexer_->getIndexReader()->hasMemBarrelReader())
                {
                    pIndexer_->setDirty();
                }<commit_msg>fix set realtime mode before add binlog to index<commit_after>#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/IndexReader.h>
#include <ir/index_manager/index/IndexBarrelWriter.h>
#include <ir/index_manager/index/FieldIndexer.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/TermPositions.h>
#include <ir/index_manager/index/EPostingWriter.h>
#include <ir/index_manager/store/IndexInput.h>

#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>
#include <util/izene_log.h>

#include <cassert>

using namespace std;

namespace bfs = boost::filesystem;

NS_IZENELIB_IR_BEGIN

namespace indexmanager
{

void writeTermInfo(
    IndexOutput* pVocWriter,
    termid_t tid,
    const TermInfo& termInfo)
{
    pVocWriter->writeInt(tid);					///write term id
    pVocWriter->writeInt(termInfo.docFreq_);		///write df
    pVocWriter->writeInt(termInfo.ctf_);			///write ctf
    pVocWriter->writeInt(termInfo.maxTF_);     ///write maxtf
    pVocWriter->writeInt(termInfo.lastDocID_);		///write last doc id
    pVocWriter->writeInt(termInfo.skipLevel_);		///write skip level
    pVocWriter->writeLong(termInfo.skipPointer_);	///write skip list offset offset
    pVocWriter->writeLong(termInfo.docPointer_);	///write document posting offset
    pVocWriter->writeInt(termInfo.docPostingLen_);	///write document posting length (without skiplist)
    pVocWriter->writeLong(termInfo.positionPointer_);	///write position posting offset
    pVocWriter->writeInt(termInfo.positionPostingLen_);///write position posting length
}

#pragma pack(push,1)
struct Record
{
    uint8_t len;
    uint32_t tid;
    uint32_t docId;
    uint32_t offset;
};
#pragma pack(pop)

FieldIndexer::FieldIndexer(
    const char* field,
    Indexer* pIndexer
)
    :pBinlog_(NULL)
    ,field_(field)
    ,pIndexer_(pIndexer)
    ,vocFilePointer_(0)
    //,alloc_(0)
    ,f_(0)
    ,termCount_(0)
    ,iHitsMax_(0)
    ,recordCount_(0)
    ,run_num_(0)
    ,pHits_(0)
    ,pHitsMax_(0)
    ,flush_(false)
{
    skipInterval_ = pIndexer_->getSkipInterval();
    maxSkipLevel_ = pIndexer_->getMaxSkipLevel();
    indexLevel_ = pIndexer_->pConfigurationManager_->indexStrategy_.indexLevel_;

    sorterFileName_ = field_+".tmp";
    bfs::path path(bfs::path(pIndexer_->pConfigurationManager_->indexStrategy_.indexLocation_)
                   /bfs::path(sorterFileName_));
    sorterFullPath_ = path.string();
}

FieldIndexer::~FieldIndexer()
{
    if (f_)
    {
        fclose(f_);
        if(! boost::filesystem::remove(sorterFullPath_))
            LOG(WARNING) << "FieldIndexer::~FieldIndexer(): failed to remove file " << sorterFullPath_;
    }

    deletebinlog();
}

void FieldIndexer::deletebinlog()
{
    boost::filesystem::remove(BinlogPath_);
}

void FieldIndexer::checkBinlog()
{
    BarrelInfo* pCurBarrelInfo = pIndexer_->getIndexWriter()->getBarrelInfo();

    if(pBinlog_ == NULL)
    {
        pBinlog_ = new Binlog(pIndexer_);
        std::string BinlogName = field_ + ".binlog";
        bfs::path path(bfs::path(pIndexer_->pConfigurationManager_->indexStrategy_.indexLocation_)
                       /bfs::path(BinlogName));
        BinlogPath_ = path.string();
        if(pBinlog_->openForRead(BinlogPath_))
        {
            if (!pIndexer_->isRealTime())
                pIndexer_->setIndexMode("realtime");
            if(!pCurBarrelInfo)
                pIndexer_->getIndexWriter()->createBarrelInfo();
            if (pIndexer_->isRealTime())
            {
                if (!pIndexer_->getIndexReader()->hasMemBarrelReader())
                {
                    pIndexer_->setDirty();
                }
{'fraction_non_alphanumeric': 0.12961421038023868, 'fraction_numerical': 0.004718290313627533, 'mean_word_length': 3.44327990135635, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6263126', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2378, 'n_words': 373}