<commit_before>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "searchiteratorverifier.h"
#include "initrange.h"
#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/searchlib/queryeval/emptysearch.h>
#include <vespa/searchlib/queryeval/truesearch.h>
#include <vespa/searchlib/queryeval/termwise_search.h>
#include <vespa/searchlib/queryeval/andsearch.h>
#include <vespa/searchlib/queryeval/orsearch.h>
#include <vespa/searchlib/common/bitvectoriterator.h>
#include <set>


namespace search {
namespace test {

using namespace search::queryeval;
using std::make_unique;

class DocIdIterator : public SearchIterator
{
public:
    DocIdIterator(const SearchIteratorVerifier::DocIds & docIds, bool strict) :
        _strict(strict),
        _currIndex(0),
        _docIds(docIds)
    { }

    void initRange(uint32_t beginId, uint32_t endId) override {
        SearchIterator::initRange(beginId, endId);
        _currIndex = 0;
        if (_strict) {
            doSeek(beginId);
        }
    }

    void doSeek(uint32_t docId) override {
        while ((_currIndex < _docIds.size()) && (_docIds[_currIndex] < docId)) {
            _currIndex++;
        }
        if ((_currIndex < _docIds.size()) && (_docIds[_currIndex] < getEndId())) {
            if (_docIds[_currIndex] == docId || _strict) {
                setDocId(_docIds[_currIndex]);
            }
        } else {
            setAtEnd();
        }
    }

    void doUnpack(uint32_t docid) { (void) docid; }

    vespalib::Trinary is_strict() const override {
        return _strict ? vespalib::Trinary::True : vespalib::Trinary::False;
    }

private:
    const bool _strict;
    uint32_t   _currIndex;
    const SearchIteratorVerifier::DocIds _docIds;
};

SearchIteratorVerifier::SearchIteratorVerifier() :
    _trueTfmd(),
    _docIds(),
    _everyOddBitSet(BitVector::create(getDocIdLimit()))
{
    for (size_t i(1); i < getDocIdLimit(); i += 2) {
        _everyOddBitSet->setBit(i);
    }
    // (0),1 and 10,11 and 20,21 .... 200,201 etc are hits
    // 0 is of course invalid.
    for (size_t i(0); (i*10+1) < getDocIdLimit(); i++) {
        if (i > 0) {
            _docIds.push_back(i * 10);
        }
        _docIds.push_back(i*10 + 1);
    }
    std::set<uint32_t> orSet;
    for (uint32_t docId : _docIds) {
        orSet.insert(docId);
    }
    _everyOddBitSet->foreach_truebit([&orSet](uint32_t docId){orSet.insert(docId);});
    for (uint32_t docId : orSet) {
        _expectedOr.push_back(docId);
    }
    for (uint32_t docId : _docIds) {
        if (_everyOddBitSet->testBit(docId)) {
            _expectedAnd.push_back(docId);
        }
    }
}

SearchIterator::UP
SearchIteratorVerifier::createIterator(const DocIds &docIds, bool strict)
{
    return make_unique<DocIdIterator>(docIds, strict);
}

SearchIterator::UP
SearchIteratorVerifier::createEmptyIterator()
{
    return make_unique<EmptySearch>();
}

SearchIterator::UP
SearchIteratorVerifier::createFullIterator() const
{
    return make_unique<TrueSearch>(_trueTfmd);
}
SearchIteratorVerifier::~SearchIteratorVerifier() { }

void
SearchIteratorVerifier::verify() const {
    TEST_DO(verifyTermwise());
    TEST_DO(verifyInitRange());
}

void
SearchIteratorVerifier::verifyTermwise() const {
    TEST_DO(verify(false));
    TEST_DO(verify(true));
}

void
SearchIteratorVerifier::verifyInitRange() const {
    InitRangeVerifier initRangeTest;
    TEST_DO(initRangeTest.verify(*create(false)));
    TEST_DO(initRangeTest.verify(*create(true)));
}

void
SearchIteratorVerifier::verify_get_hits(bool strict) const {
    constexpr const size_t FIRST_LEGAL = 64;
    SearchIterator::UP iterator = create(strict);
    iterator->initFullRange();
    EXPECT_TRUE(iterator->seek(FIRST_LEGAL));
    EXPECT_EQUAL(FIRST_LEGAL, iterator->getDocId());
    BitVector::UP hits = iterator->get_hits(1);
    for (size_t i(0); i < FIRST_LEGAL; i++) {
        EXPECT_FALSE(hits->testBit(i));
    }
    EXPECT_TRUE(hits->testBit(FIRST_LEGAL));
}

void
SearchIteratorVerifier::verify(bool strict) const {
    SearchIterator::UP iterator = create(strict);
    TEST_DO(verify(*iterator, strict, _docIds));
    TEST_DO(verifyTermwise(std::move(iterator), strict, _docIds));
    TEST_DO(verifyAnd(strict));
    TEST_DO(verifyOr(strict));
    TEST_DO(verify_get_hits(strict));
}

void
SearchIteratorVerifier::verifyAnd(bool strict) const {
    fef::TermFieldMatchData tfmd;
    MultiSearch::Children children;
    children.emplace_back(create(strict).release());
    children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, false).release());
    SearchIterator::UP search(AndSearch::create(children, strict, UnpackInfo()));<commit_msg>Use a docid that is actually a hit.<commit_after>// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "searchiteratorverifier.h"
#include "initrange.h"
#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/searchlib/queryeval/emptysearch.h>
#include <vespa/searchlib/queryeval/truesearch.h>
#include <vespa/searchlib/queryeval/termwise_search.h>
#include <vespa/searchlib/queryeval/andsearch.h>
#include <vespa/searchlib/queryeval/orsearch.h>
#include <vespa/searchlib/common/bitvectoriterator.h>
#include <set>


namespace search {
namespace test {

using namespace search::queryeval;
using std::make_unique;

class DocIdIterator : public SearchIterator
{
public:
    DocIdIterator(const SearchIteratorVerifier::DocIds & docIds, bool strict) :
        _strict(strict),
        _currIndex(0),
        _docIds(docIds)
    { }

    void initRange(uint32_t beginId, uint32_t endId) override {
        SearchIterator::initRange(beginId, endId);
        _currIndex = 0;
        if (_strict) {
            doSeek(beginId);
        }
    }

    void doSeek(uint32_t docId) override {
        while ((_currIndex < _docIds.size()) && (_docIds[_currIndex] < docId)) {
            _currIndex++;
        }
        if ((_currIndex < _docIds.size()) && (_docIds[_currIndex] < getEndId())) {
            if (_docIds[_currIndex] == docId || _strict) {
                setDocId(_docIds[_currIndex]);
            }
        } else {
            setAtEnd();
        }
    }

    void doUnpack(uint32_t docid) { (void) docid; }

    vespalib::Trinary is_strict() const override {
        return _strict ? vespalib::Trinary::True : vespalib::Trinary::False;
    }

private:
    const bool _strict;
    uint32_t   _currIndex;
    const SearchIteratorVerifier::DocIds _docIds;
};

SearchIteratorVerifier::SearchIteratorVerifier() :
    _trueTfmd(),
    _docIds(),
    _everyOddBitSet(BitVector::create(getDocIdLimit()))
{
    for (size_t i(1); i < getDocIdLimit(); i += 2) {
        _everyOddBitSet->setBit(i);
    }
    // (0),1 and 10,11 and 20,21 .... 200,201 etc are hits
    // 0 is of course invalid.
    for (size_t i(0); (i*10+1) < getDocIdLimit(); i++) {
        if (i > 0) {
            _docIds.push_back(i * 10);
        }
        _docIds.push_back(i*10 + 1);
    }
    std::set<uint32_t> orSet;
    for (uint32_t docId : _docIds) {
        orSet.insert(docId);
    }
    _everyOddBitSet->foreach_truebit([&orSet](uint32_t docId){orSet.insert(docId);});
    for (uint32_t docId : orSet) {
        _expectedOr.push_back(docId);
    }
    for (uint32_t docId : _docIds) {
        if (_everyOddBitSet->testBit(docId)) {
            _expectedAnd.push_back(docId);
        }
    }
}

SearchIterator::UP
SearchIteratorVerifier::createIterator(const DocIds &docIds, bool strict)
{
    return make_unique<DocIdIterator>(docIds, strict);
}

SearchIterator::UP
SearchIteratorVerifier::createEmptyIterator()
{
    return make_unique<EmptySearch>();
}

SearchIterator::UP
SearchIteratorVerifier::createFullIterator() const
{
    return make_unique<TrueSearch>(_trueTfmd);
}
SearchIteratorVerifier::~SearchIteratorVerifier() { }

void
SearchIteratorVerifier::verify() const {
    TEST_DO(verifyTermwise());
    TEST_DO(verifyInitRange());
}

void
SearchIteratorVerifier::verifyTermwise() const {
    TEST_DO(verify(false));
    TEST_DO(verify(true));
}

void
SearchIteratorVerifier::verifyInitRange() const {
    InitRangeVerifier initRangeTest;
    TEST_DO(initRangeTest.verify(*create(false)));
    TEST_DO(initRangeTest.verify(*create(true)));
}

void
SearchIteratorVerifier::verify_get_hits(bool strict) const {
    constexpr const size_t FIRST_LEGAL = 61;
    SearchIterator::UP iterator = create(strict);
    iterator->initFullRange();
    EXPECT_TRUE(iterator->seek(FIRST_LEGAL));
    EXPECT_EQUAL(FIRST_LEGAL, iterator->getDocId());
    BitVector::UP hits = iterator->get_hits(1);
    for (size_t i(0); i < FIRST_LEGAL; i++) {
        EXPECT_FALSE(hits->testBit(i));
    }
    EXPECT_TRUE(hits->testBit(FIRST_LEGAL));
}

void
SearchIteratorVerifier::verify(bool strict) const {
    SearchIterator::UP iterator = create(strict);
    TEST_DO(verify(*iterator, strict, _docIds));
    TEST_DO(verifyTermwise(std::move(iterator), strict, _docIds));
    TEST_DO(verifyAnd(strict));
    TEST_DO(verifyOr(strict));
    TEST_DO(verify_get_hits(strict));
}

void
SearchIteratorVerifier::verifyAnd(bool strict) const {
    fef::TermFieldMatchData tfmd;
    MultiSearch::Children children;
    children.emplace_back(create(strict).release());
    children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, false).release());
    SearchIterator::UP search(AndSearch::create(children, strict, UnpackInfo()));
{'fraction_non_alphanumeric': 0.13114925685576723, 'fraction_numerical': 0.012769520619635755, 'mean_word_length': 3.9075500770416025, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7058093', 'n_tokens_mistral': 3255, 'n_tokens_neox': 3146, 'n_words': 556}