<commit_before>#include "ORDocumentIterator.h"
#include "NOTDocumentIterator.h"

using namespace std;
using namespace sf1r;

ORDocumentIterator::ORDocumentIterator()
        :pDocIteratorQueue_(NULL)
        ,hasNot_(false)
        ,currDocOfNOTIter_(MAX_DOC_ID)
        ,initNOTIterator_(false)
        ,pNOTDocIterator_(NULL)
{
}

ORDocumentIterator::~ORDocumentIterator()
{
    vector<DocumentIterator*>::iterator iter = docIteratorList_.begin();
    for (; iter != docIteratorList_.end(); ++iter)
        if(*iter) delete *iter;

    if (pNOTDocIterator_)
        delete pNOTDocIterator_;

    if (pDocIteratorQueue_)
        delete pDocIteratorQueue_;
}

void ORDocumentIterator::initDocIteratorQueue()
{
    if (docIteratorList_.size() < 1)
        return;
    pDocIteratorQueue_ = new DocumentIteratorQueue(docIteratorList_.size());
    DocumentIterator* pDocIterator;
    vector<DocumentIterator*>::iterator iter = docIteratorList_.begin();
    while (iter != docIteratorList_.end())
    {
        pDocIterator = (*iter);
        if (pDocIterator)
        {
            pDocIterator->setCurrent(false);
            if(pDocIterator->next())
                pDocIteratorQueue_->insert(pDocIterator);
           else
           {
               ///Mark here!
               ///If a DocumentIterator does not contain member, remove it from docIteratorList_
               *iter = NULL;
               delete pDocIterator;
           }
        }
        iter ++;
    }
}

void ORDocumentIterator::add(DocumentIterator* pDocIterator)
{
    if (pDocIterator->isNot())
    {
        hasNot_ = true;
        if (NULL == pNOTDocIterator_)
            pNOTDocIterator_ = new NOTDocumentIterator();
        pNOTDocIterator_->add(pDocIterator);
    }
    else
        docIteratorList_.push_back(pDocIterator);
}

bool ORDocumentIterator::next()
{
    if (!hasNot_)
        return do_next();
    else
    {
        if (! initNOTIterator_)
        {
            initNOTIterator_ = true;
            if (pNOTDocIterator_->next())
                currDocOfNOTIter_ = pNOTDocIterator_->doc();
            else
                currDocOfNOTIter_ = MAX_DOC_ID;
        }

        bool ret = do_next();

        if (currDoc_ == currDocOfNOTIter_)
            return move_together_with_not();
        if (currDoc_ < currDocOfNOTIter_)
            return ret;
        else
        {
            currDocOfNOTIter_ = pNOTDocIterator_->skipTo(currDoc_);
            if (currDoc_ == currDocOfNOTIter_)
                return move_together_with_not();
            else
                return ret;
        }
    }
}

bool ORDocumentIterator::move_together_with_not()
{
    bool ret;
    do
    {
        ret = do_next();
        if (pNOTDocIterator_->next())
            currDocOfNOTIter_ = pNOTDocIterator_->doc();<commit_msg>refactor ORDocumentIterator::next().<commit_after>#include "ORDocumentIterator.h"
#include "NOTDocumentIterator.h"

using namespace std;
using namespace sf1r;

ORDocumentIterator::ORDocumentIterator()
        :pDocIteratorQueue_(NULL)
        ,hasNot_(false)
        ,currDocOfNOTIter_(MAX_DOC_ID)
        ,initNOTIterator_(false)
        ,pNOTDocIterator_(NULL)
{
}

ORDocumentIterator::~ORDocumentIterator()
{
    vector<DocumentIterator*>::iterator iter = docIteratorList_.begin();
    for (; iter != docIteratorList_.end(); ++iter)
        if(*iter) delete *iter;

    if (pNOTDocIterator_)
        delete pNOTDocIterator_;

    if (pDocIteratorQueue_)
        delete pDocIteratorQueue_;
}

void ORDocumentIterator::initDocIteratorQueue()
{
    if (docIteratorList_.size() < 1)
        return;
    pDocIteratorQueue_ = new DocumentIteratorQueue(docIteratorList_.size());
    DocumentIterator* pDocIterator;
    vector<DocumentIterator*>::iterator iter = docIteratorList_.begin();
    while (iter != docIteratorList_.end())
    {
        pDocIterator = (*iter);
        if (pDocIterator)
        {
            pDocIterator->setCurrent(false);
            if(pDocIterator->next())
                pDocIteratorQueue_->insert(pDocIterator);
           else
           {
               ///Mark here!
               ///If a DocumentIterator does not contain member, remove it from docIteratorList_
               *iter = NULL;
               delete pDocIterator;
           }
        }
        iter ++;
    }
}

void ORDocumentIterator::add(DocumentIterator* pDocIterator)
{
    if (pDocIterator->isNot())
    {
        hasNot_ = true;
        if (NULL == pNOTDocIterator_)
            pNOTDocIterator_ = new NOTDocumentIterator();
        pNOTDocIterator_->add(pDocIterator);
    }
    else
        docIteratorList_.push_back(pDocIterator);
}

bool ORDocumentIterator::next()
{
    if (!hasNot_)
        return do_next();

    if (! initNOTIterator_)
    {
        initNOTIterator_ = true;
        if (pNOTDocIterator_->next())
            currDocOfNOTIter_ = pNOTDocIterator_->doc();
        else
            currDocOfNOTIter_ = MAX_DOC_ID;
    }

    bool ret = do_next();

    if (currDoc_ > currDocOfNOTIter_)
        currDocOfNOTIter_ = pNOTDocIterator_->skipTo(currDoc_);

    if (currDoc_ == currDocOfNOTIter_)
        return move_together_with_not();

    assert(currDoc_ < currDocOfNOTIter_);
    return ret;
}

bool ORDocumentIterator::move_together_with_not()
{
    bool ret;
    do
    {
        ret = do_next();
        if (pNOTDocIterator_->next())
            currDocOfNOTIter_ = pNOTDocIterator_->doc();
{'fraction_non_alphanumeric': 0.10234187718974737, 'fraction_numerical': 0.0007375991148810622, 'mean_word_length': 2.39, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '128789', 'n_tokens_mistral': 1567, 'n_tokens_neox': 1544, 'n_words': 286}