<commit_before>#include <ir/index_manager/index/MultiFieldTermReader.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/FieldInfo.h>

#include <memory> // auto_ptr

using namespace izenelib::ir::indexmanager;

MultiFieldTermReader::MultiFieldTermReader(Directory* pDirectory,BarrelInfo* pBarrelInfo,FieldsInfo* pFieldsInfo)
        : TermReader()
        , pCurReader_(NULL)
{
    FieldInfo* pInfo = NULL;
    TermReader* pTermReader = NULL;
    pFieldsInfo->startIterator();
    while (pFieldsInfo->hasNext())
    {
        pInfo = pFieldsInfo->next();

        if (pInfo->isIndexed()&&pInfo->isAnalyzed())
        {
            switch(pBarrelInfo->compressType)
            {
            case BYTEALIGN:
                pTermReader = new RTDiskTermReader(pDirectory,pBarrelInfo,pInfo);
                break;
            case BLOCK:
                pTermReader = new BlockTermReader(pDirectory,pBarrelInfo,pInfo);
                break;
            case CHUNK:
                pTermReader = new ChunkTermReader(pDirectory,pBarrelInfo,pInfo);
                break;
            default:
                assert(false);
            }
            fieldsTermReaders_.insert(pair<string,TermReader*>(pInfo->getName(),pTermReader));
        }
    }
}

MultiFieldTermReader::MultiFieldTermReader()
        : TermReader()
        , pCurReader_(NULL)
{
}

MultiFieldTermReader::~MultiFieldTermReader()
{
    close();
}

void MultiFieldTermReader::setDocFilter(BitVector* pFilter) 
{
    for(map<string,TermReader*>::iterator iter = fieldsTermReaders_.begin();
        iter != fieldsTermReaders_.end(); ++iter)
        iter->second->setDocFilter(pFilter);
}

void MultiFieldTermReader::open(Directory* pDirectory,BarrelInfo* pBarrelInfo,FieldInfo* pFieldInfo)
{}

void MultiFieldTermReader::reopen()
{
    for(reader_map::iterator iter = fieldsTermReaders_.begin(); 
            iter != fieldsTermReaders_.end(); ++iter)
        iter->second->reopen();
}

TermReader* MultiFieldTermReader::termReader(const char* field)
{
    reader_map::iterator iter = fieldsTermReaders_.find(field);
    if (iter != fieldsTermReaders_.end())
    {
        return iter->second;
    }
    return NULL;
}


TermIterator* MultiFieldTermReader::termIterator(const char* field)
{
    map<string,TermReader*>::iterator iter = fieldsTermReaders_.find(field);
    if (iter != fieldsTermReaders_.end())
        return iter->second->termIterator(field);
    return NULL;
}

bool MultiFieldTermReader::seek( Term* term)
{
    string field = term->getField();
    map<string,TermReader*>::iterator iter = fieldsTermReaders_.find(field);
    if (iter != fieldsTermReaders_.end())
    {
        pCurReader_ = iter->second;
        return pCurReader_->seek(term);
    }
    return false;
}

TermDocFreqs* MultiFieldTermReader::termDocFreqs()
{
    return pCurReader_->termDocFreqs();
}

TermPositions* MultiFieldTermReader::termPositions()
{
    return pCurReader_->termPositions();
}


freq_t MultiFieldTermReader::docFreq(Term* term)
{
    return pCurReader_->docFreq(term);
}

TermInfo* MultiFieldTermReader::termInfo(Term* term)
{
    return pCurReader_->termInfo(term);
}

void MultiFieldTermReader::close()
{
    pCurReader_ = NULL;;

    reader_map::iterator iter = fieldsTermReaders_.begin();
    while (iter != fieldsTermReaders_.end())
    {
        delete iter->second;
        iter++;
    }
    fieldsTermReaders_.clear();
}<commit_msg>in MultiFieldTermReader::docFreq() and termInfo(), instead of using pCurReader_, get TermReader from parameter "term".<commit_after>#include <ir/index_manager/index/MultiFieldTermReader.h>
#include <ir/index_manager/index/TermReader.h>
#include <ir/index_manager/index/FieldInfo.h>

#include <memory> // auto_ptr

using namespace izenelib::ir::indexmanager;

MultiFieldTermReader::MultiFieldTermReader(Directory* pDirectory,BarrelInfo* pBarrelInfo,FieldsInfo* pFieldsInfo)
        : TermReader()
        , pCurReader_(NULL)
{
    FieldInfo* pInfo = NULL;
    TermReader* pTermReader = NULL;
    pFieldsInfo->startIterator();
    while (pFieldsInfo->hasNext())
    {
        pInfo = pFieldsInfo->next();

        if (pInfo->isIndexed()&&pInfo->isAnalyzed())
        {
            switch(pBarrelInfo->compressType)
            {
            case BYTEALIGN:
                pTermReader = new RTDiskTermReader(pDirectory,pBarrelInfo,pInfo);
                break;
            case BLOCK:
                pTermReader = new BlockTermReader(pDirectory,pBarrelInfo,pInfo);
                break;
            case CHUNK:
                pTermReader = new ChunkTermReader(pDirectory,pBarrelInfo,pInfo);
                break;
            default:
                assert(false);
            }
            fieldsTermReaders_.insert(pair<string,TermReader*>(pInfo->getName(),pTermReader));
        }
    }
}

MultiFieldTermReader::MultiFieldTermReader()
        : TermReader()
        , pCurReader_(NULL)
{
}

MultiFieldTermReader::~MultiFieldTermReader()
{
    close();
}

void MultiFieldTermReader::setDocFilter(BitVector* pFilter) 
{
    for(map<string,TermReader*>::iterator iter = fieldsTermReaders_.begin();
        iter != fieldsTermReaders_.end(); ++iter)
        iter->second->setDocFilter(pFilter);
}

void MultiFieldTermReader::open(Directory* pDirectory,BarrelInfo* pBarrelInfo,FieldInfo* pFieldInfo)
{}

void MultiFieldTermReader::reopen()
{
    for(reader_map::iterator iter = fieldsTermReaders_.begin(); 
            iter != fieldsTermReaders_.end(); ++iter)
        iter->second->reopen();
}

TermReader* MultiFieldTermReader::termReader(const char* field)
{
    reader_map::iterator iter = fieldsTermReaders_.find(field);
    if (iter != fieldsTermReaders_.end())
    {
        return iter->second;
    }
    return NULL;
}


TermIterator* MultiFieldTermReader::termIterator(const char* field)
{
    map<string,TermReader*>::iterator iter = fieldsTermReaders_.find(field);
    if (iter != fieldsTermReaders_.end())
        return iter->second->termIterator(field);
    return NULL;
}

bool MultiFieldTermReader::seek( Term* term)
{
    string field = term->getField();
    map<string,TermReader*>::iterator iter = fieldsTermReaders_.find(field);
    if (iter != fieldsTermReaders_.end())
    {
        pCurReader_ = iter->second;
        return pCurReader_->seek(term);
    }
    return false;
}

TermDocFreqs* MultiFieldTermReader::termDocFreqs()
{
    return pCurReader_->termDocFreqs();
}

TermPositions* MultiFieldTermReader::termPositions()
{
    return pCurReader_->termPositions();
}


freq_t MultiFieldTermReader::docFreq(Term* term)
{
    if(TermReader* pTermReader = termReader(term->getField()))
        return pTermReader->docFreq(term);
    return NULL;
}

TermInfo* MultiFieldTermReader::termInfo(Term* term)
{
    if(TermReader* pTermReader = termReader(term->getField()))
        return pTermReader->termInfo(term);
    return NULL;
}

void MultiFieldTermReader::close()
{
    pCurReader_ = NULL;;

    reader_map::iterator iter = fieldsTermReaders_.begin();
    while (iter != fieldsTermReaders_.end())
    {
        delete iter->second;
        iter++;
    }
    fieldsTermReaders_.clear();
}
{'fraction_non_alphanumeric': 0.12368972746331237, 'fraction_numerical': 0.0, 'mean_word_length': 3.4118372379778052, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1367283', 'n_tokens_mistral': 2175, 'n_tokens_neox': 2153, 'n_words': 344}