<commit_before>#include <ir/index_manager/index/IndexReader.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/MultiIndexBarrelReader.h>
#include <util/izene_log.h>

#include <util/ThreadModel.h>

using namespace izenelib::ir::indexmanager;

IndexReader::IndexReader(Indexer* pIndex)
        :pIndexer_(pIndex)
        ,pBarrelsInfo_(NULL)
        ,pBarrelReader_(NULL)
        ,pDocFilter_(NULL)
        ,pDocLengthReader_(NULL)
{
    pBarrelsInfo_ = pIndexer_->getBarrelsInfo();

    Directory* pDirectory = pIndexer_->getDirectory();
    if(pDirectory->fileExists(DELETED_DOCS))
    {
        pDocFilter_ = new BitVector;
        pDocFilter_->read(pDirectory, DELETED_DOCS);
    }
    ///todo since only one collection is used within indexmanager, so we only get collectionmeta for one collection to build
    ///up the DocLengthReader
    if(pIndexer_->getIndexManagerConfig()->indexStrategy_.indexDocLength_)
        pDocLengthReader_ = new DocLengthReader(
                                pIndexer_->getCollectionsMeta().begin()->second.getDocumentSchema(), 
                                pIndexer_->getDirectory());
}

IndexReader::~IndexReader()
{
    if (pBarrelReader_)
    {
        delete pBarrelReader_;
        pBarrelReader_ = NULL;
    }

    flush();
    delete pDocFilter_;
    pDocFilter_ = NULL;

    if(pDocLengthReader_)
    {
        delete pDocLengthReader_;
        pDocLengthReader_ = NULL;
    }
}

void IndexReader::delDocFilter()
{
    DVLOG(2) << "=> IndexReader::delDocFilter(), pDocFilter_: " << pDocFilter_;
    if(pDocFilter_)
    {
        if(pIndexer_->getDirectory()->fileExists(DELETED_DOCS))
            pIndexer_->getDirectory()->deleteFile(DELETED_DOCS);
        pDocFilter_->clear();
    }
    DVLOG(2) << "<= IndexReader::delDocFilter()";
}

void IndexReader::flush()
{
    if(pDocFilter_)
    {
        boost::mutex::scoped_lock docFilterLock(docFilterMutex_);

        if(pDocFilter_->any())
            pDocFilter_->write(pIndexer_->getDirectory(), DELETED_DOCS);
    }
}

docid_t IndexReader::maxDoc()
{
    return pBarrelsInfo_->maxDocId();
}

size_t IndexReader::docLength(docid_t docId, fieldid_t fid)
{
    if (pBarrelReader_ == NULL)
        createBarrelReader();
    return pDocLengthReader_->docLength(docId, fid);
}

double IndexReader::getAveragePropertyLength(fieldid_t fid)
{
    if (pBarrelReader_ == NULL)
        createBarrelReader();
    boost::mutex::scoped_lock indexReaderLock(this->mutex_);
    return pDocLengthReader_->averagePropertyLength(fid);
}

void IndexReader::createBarrelReader()
{
    boost::mutex::scoped_lock lock(this->mutex_);

    if (pBarrelReader_)
        return;

    DVLOG(2) << "=> IndexReader::createBarrelReader()";

    if(int32_t bc = pBarrelsInfo_->getBarrelCount())
    {
        DVLOG(2) << "IndexReader::createBarrelReader() => " << bc << " barrels...";

        pBarrelReader_ = new MultiIndexBarrelReader(this, pBarrelsInfo_);

        if(pDocLengthReader_)
            pDocLengthReader_->load(pBarrelsInfo_->maxDocId());
    }

    DVLOG(2) << "<= IndexReader::createBarrelReader()";
}

TermReader* IndexReader::getTermReader(collectionid_t colID)
{
    //boost::try_mutex::scoped_try_lock lock(pIndexer_->mutex_);
    //if(!lock.owns_lock())
        //return NULL;
    izenelib::util::ScopedReadLock<izenelib::util::ReadWriteLock> lock(pIndexer_->mutex_);
    if (pBarrelReader_ == NULL)
        createBarrelReader();
    if (pBarrelReader_ == NULL)
        return NULL;
    boost::mutex::scoped_lock indexReaderLock(this->mutex_);
	
    TermReader* pTermReader = pBarrelReader_->termReader(colID);
    if (pTermReader)
    {
        pTermReader->setSkipInterval(pIndexer_->getSkipInterval());
        pTermReader->setMaxSkipLevel(pIndexer_->getMaxSkipLevel());
        if(pDocFilter_)<commit_msg>undo a previous commit in 9d283a93b5825c25611eff37459ea87794d03df3, that is, in IndexReader::createBarrelReader(), instead of always using MultiIndexBarrelReader, use SingleIndexBarrelReader when there is only one barrel.<commit_after>#include <ir/index_manager/index/IndexReader.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/Indexer.h>
#include <ir/index_manager/index/MultiIndexBarrelReader.h>
#include <ir/index_manager/index/IndexBarrelWriter.h>
#include <util/izene_log.h>

#include <util/ThreadModel.h>

using namespace izenelib::ir::indexmanager;

IndexReader::IndexReader(Indexer* pIndex)
        :pIndexer_(pIndex)
        ,pBarrelsInfo_(NULL)
        ,pBarrelReader_(NULL)
        ,pDocFilter_(NULL)
        ,pDocLengthReader_(NULL)
{
    pBarrelsInfo_ = pIndexer_->getBarrelsInfo();

    Directory* pDirectory = pIndexer_->getDirectory();
    if(pDirectory->fileExists(DELETED_DOCS))
    {
        pDocFilter_ = new BitVector;
        pDocFilter_->read(pDirectory, DELETED_DOCS);
    }
    ///todo since only one collection is used within indexmanager, so we only get collectionmeta for one collection to build
    ///up the DocLengthReader
    if(pIndexer_->getIndexManagerConfig()->indexStrategy_.indexDocLength_)
        pDocLengthReader_ = new DocLengthReader(
                                pIndexer_->getCollectionsMeta().begin()->second.getDocumentSchema(), 
                                pIndexer_->getDirectory());
}

IndexReader::~IndexReader()
{
    if (pBarrelReader_)
    {
        delete pBarrelReader_;
        pBarrelReader_ = NULL;
    }

    flush();
    delete pDocFilter_;
    pDocFilter_ = NULL;

    if(pDocLengthReader_)
    {
        delete pDocLengthReader_;
        pDocLengthReader_ = NULL;
    }
}

void IndexReader::delDocFilter()
{
    DVLOG(2) << "=> IndexReader::delDocFilter(), pDocFilter_: " << pDocFilter_;
    if(pDocFilter_)
    {
        if(pIndexer_->getDirectory()->fileExists(DELETED_DOCS))
            pIndexer_->getDirectory()->deleteFile(DELETED_DOCS);
        pDocFilter_->clear();
    }
    DVLOG(2) << "<= IndexReader::delDocFilter()";
}

void IndexReader::flush()
{
    if(pDocFilter_)
    {
        boost::mutex::scoped_lock docFilterLock(docFilterMutex_);

        if(pDocFilter_->any())
            pDocFilter_->write(pIndexer_->getDirectory(), DELETED_DOCS);
    }
}

docid_t IndexReader::maxDoc()
{
    return pBarrelsInfo_->maxDocId();
}

size_t IndexReader::docLength(docid_t docId, fieldid_t fid)
{
    if (pBarrelReader_ == NULL)
        createBarrelReader();
    return pDocLengthReader_->docLength(docId, fid);
}

double IndexReader::getAveragePropertyLength(fieldid_t fid)
{
    if (pBarrelReader_ == NULL)
        createBarrelReader();
    boost::mutex::scoped_lock indexReaderLock(this->mutex_);
    return pDocLengthReader_->averagePropertyLength(fid);
}

void IndexReader::createBarrelReader()
{
    boost::mutex::scoped_lock lock(this->mutex_);

    if(pBarrelReader_)
        return;

    DVLOG(2) << "=> IndexReader::createBarrelReader()";

    if(int32_t bc = pBarrelsInfo_->getBarrelCount())
    {
        DVLOG(2) << "IndexReader::createBarrelReader() => " << bc << " barrels...";

        if(bc == 1)
        {
            boost::mutex::scoped_lock lock(pBarrelsInfo_->getMutex());

            pBarrelsInfo_->startIterator();
            if(pBarrelsInfo_->hasNext())
            {
                BarrelInfo* pBarrelInfo = pBarrelsInfo_->next();
                if(pBarrelInfo->isSearchable() && pBarrelInfo->getDocCount() > 0)
                {
                    if(IndexBarrelWriter* pBarrelWriter = pBarrelInfo->getWriter())
                        pBarrelReader_ = pBarrelWriter->inMemoryReader();
                    else
                        pBarrelReader_ = new SingleIndexBarrelReader(this, pBarrelInfo);
                }
            }
        }
        else if(bc > 1)
            pBarrelReader_ = new MultiIndexBarrelReader(this, pBarrelsInfo_);

        if(pDocLengthReader_)
            pDocLengthReader_->load(pBarrelsInfo_->maxDocId());
    }

    DVLOG(2) << "<= IndexReader::createBarrelReader()";
}

TermReader* IndexReader::getTermReader(collectionid_t colID)
{
    //boost::try_mutex::scoped_try_lock lock(pIndexer_->mutex_);
    //if(!lock.owns_lock())
        //return NULL;
    izenelib::util::ScopedReadLock<izenelib::util::ReadWriteLock> lock(pIndexer_->mutex_);
    if (pBarrelReader_ == NULL)
        createBarrelReader();
    if (pBarrelReader_ == NULL)
        return NULL;
    boost::mutex::scoped_lock indexReaderLock(this->mutex_);
	
    TermReader* pTermReader = pBarrelReader_->termReader(colID);
    if (pTermReader)
    {
        pTermReader->setSkipInterval(pIndexer_->getSkipInterval());
        pTermReader->setMaxSkipLevel(pIndexer_->getMaxSkipLevel());
        if(pDocFilter_)
{'fraction_non_alphanumeric': 0.12521499828001376, 'fraction_numerical': 0.005159958720330237, 'mean_word_length': 3.3285359801488834, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 100, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5386699', 'n_tokens_mistral': 2823, 'n_tokens_neox': 2733, 'n_words': 403}