<commit_before>/**
 * @file        BTreeIndex.h
 * @author     <NAME>
 * @version     SF1 v5.0
 * @brief head file of BTree Indexer, it is an enclosion of wiselib::IndexSDB
 */

#ifndef BTREEINDEX_H
#define BTREEINDEX_H

#include <ir/index_manager/index/IndexerDocument.h>
#include <ir/index_manager/utility/BitVector.h>

#include <sdb/SequentialDB.h>
#include <sdb/IndexSDB.h>
#include <sdb/TrieIndexSDB.h>
#include <util/BoostVariantUtil.h>

#include <boost/variant.hpp>
#include <boost/utility/enable_if.hpp>
#include <boost/type_traits.hpp>
#include <boost/mpl/bool.hpp>
#include <boost/mpl/not.hpp>
#include <boost/bind.hpp>
#include <functional>

#include <algorithm>
#include <string>

NS_IZENELIB_IR_BEGIN

namespace indexmanager
{

template<class T>
struct IndexKeyType
{
    collectionid_t cid;
    fieldid_t fid;
    T value;

    IndexKeyType():cid(0),fid(0),value(T()) {}

    IndexKeyType(collectionid_t c, fieldid_t f, T& v)
            :cid(c), fid(f), value(v)
    {}
    template<class Archive> void serialize(Archive & ar,const unsigned int version)
    {
        ar & fid;
        ar & cid;
        ar & value;
    }

    int compare(const IndexKeyType<T>& other) const
    {
        if (cid != other.cid)
            return cid-other.cid;
        else
            if (fid != other.fid)
                return fid-other.fid;
            else
                return __compare(other,static_cast<boost::is_arithmetic<T>*>(0));
    }

    bool isPrefix(const IndexKeyType<T>& other) const
    {
        if (cid != other.cid)
            return false;
        else
        {
            if (fid != other.fid)
                return false;
            else
            {
                String str;
                other.value.substr(str, 0, value.length());
                if (str == value)
                    return true;
                else
                    return false;
            }
        }
    }

    void display() const
    {
        cout<<cid<<" ";
        cout<<fid<<" ";
        //cout<<value;
    }
    bool sameField(const IndexKeyType<T>& other) const
    {
        return ((cid == other.cid) && (fid == other.fid));
    }

private:
    int __compare(const IndexKeyType<T>& other, const boost::mpl::true_*) const
    {
        if ( value> other.value)
        {
            return 1;
        }
        else if (value < other.value)
        {
            return -1;
        }
        else
            return 0;
        //return (int)(value - other.value);
    }
    int __compare(const IndexKeyType<T>& other, const boost::mpl::false_*) const
    {
        return value.compare(other.value);
    }
};

/**
 * BTreeIndex
 */

template <class KeyType>
class BTreeIndex:public izenelib::sdb::IndexSDB<KeyType,docid_t>
{
    typedef izenelib::sdb::iKeyType<KeyType> myKeyType;
    typedef std::vector<docid_t> myValueType;
    typedef DataType<myKeyType, myValueType> myDataType;
public:
    typedef typename IndexSDB<KeyType,docid_t>::IndexSDBCursor IndexSDBCursor;

public:

    BTreeIndex(string& fileName) :
            izenelib::sdb::IndexSDB<KeyType,docid_t>(fileName) //0, use default compare
    {
    }
    bool remove(const KeyType& key, docid_t docID);

    template<typename SDBCursor>
    bool search(const KeyType& key, SDBCursor& locn)
    {
        izenelib::sdb::iKeyType<KeyType> ikey(key, 0);
        return this->_sdb.search(ikey, locn);
    }

    template<typename SDBCursor>
    bool seq(SDBCursor& locn, KeyType& key, std::vector<docid_t>& value, ESeqDirection sdir)
    {
        izenelib::sdb::iKeyType<KeyType> ikey(key, 0);
        return this->_sdb.seq(locn, ikey, value, sdir);
    }

    bool get(const KeyType& key,BitVector& result)
    {
        myValueType vdat;
        unsigned int offset = 0;
        myKeyType firstKey(key, 0);
        if ( !this->_sdb.hasKey(firstKey) )
            return false;
        do
        {
            myKeyType ikey(key, offset++);
            if (this->_sdb.getValue(ikey, vdat))
                for (size_t i=0; i<vdat.size(); i++)
                    result.set(vdat[i]);
            else
                break;
        }
        while (true);
        return true;
    }

    bool get_without(const KeyType& key,BitVector& result)
    {
        myValueType vdat;
        unsigned int offset = 0;
        myKeyType firstKey(key, 0);
        if ( !this->_sdb.hasKey(firstKey) )
            return false;
        result.setAll();
        do
        {
            myKeyType ikey(key, offset++);
            if (this->_sdb.getValue(ikey, vdat))
                for (size_t i=0; i<vdat.size(); i++)
                    result.clear(vdat[i]);
            else
                break;
        }
        while (true);
        return true;
    }

    void getGreat(const KeyType& key, BitVector& result)
    {
        KeyType temp = getNext(key);
        myKeyType ikey(temp, 0);
        IndexSDBCursor locn;
        this->_sdb.search(ikey, locn);
        myDataType rec;
        while (this->_sdb.seq(locn, ESD_FORWARD) )
        {
            if (this->_sdb.get(locn, rec))
            {
                vector<docid_t> vdat = rec.get_value();
                for (size_t i=0; i<vdat.size(); i++)
                    result.set(vdat[i]);
            }
        }

    }

    void getGreatEqual(const KeyType& key, BitVector& result)
    {
        get(key, result);
        getGreat(key, result);
    }

    void getLess(const KeyType& key, BitVector& result)
    {
        KeyType temp = getPrev(key);
        myKeyType ikey(temp, 0);
        IndexSDBCursor locn;
        this->_sdb.search(ikey, locn);
        myDataType rec;
        while (this->_sdb.seq(locn, ESD_BACKWARD) )
        {
            if (this->_sdb.get(locn, rec) )
            {
                vector<docid_t> vdat = rec.get_value();
                for (size_t i=0; i<vdat.size(); i++)
                    result.set(vdat[i]);
            }
        }

    }

    void getLessEqual(const KeyType& key, BitVector& result)
    {
        get(key, result);
        getLess(key, result);
    }

    void getPrefix(const KeyType& key, BitVector& result)
    {
        myKeyType ikey(key, 0);
        myKeyType temp;
        myDataType idat;
        temp = this->_sdb.getNearest(ikey);
        IndexSDBCursor locn;
        this->_sdb.search(temp, locn);
        while ( ikey.isPrefix(temp) )
        {
            vector<docid_t>& vdat = idat.get_value();
            for (size_t i=0; i<vdat.size(); i++)
                result.set(vdat[i]);
            if (this->_sdb.seq(locn, ESD_FORWARD) )
                if (this->_sdb.get(locn, idat))
                    temp = idat.get_key();
                else
                    break;
        }
    }

};

template <class KeyType>
bool BTreeIndex<KeyType>::remove(const KeyType& key, docid_t docID)
{
    vector<docid_t> vDat;
    getValue(key, vDat);

    //if we can make sure there are no duplicate inserting items, e.g, documentID
    //can't be the same as the existing items in IndexSDB, i.e nodup is true.
    vector<docid_t>::iterator iter = find(vDat.begin(), vDat.end(), docID);
    if (iter != vDat.end() )
    {
        vDat.erase(iter);<commit_msg>prefix can work due to merge problem..<commit_after>/**
 * @file        BTreeIndex.h
 * @author     <NAME>
 * @version     SF1 v5.0
 * @brief head file of BTree Indexer, it is an enclosion of wiselib::IndexSDB
 */

#ifndef BTREEINDEX_H
#define BTREEINDEX_H

#include <ir/index_manager/index/IndexerDocument.h>
#include <ir/index_manager/utility/BitVector.h>

#include <sdb/SequentialDB.h>
#include <sdb/IndexSDB.h>
#include <sdb/TrieIndexSDB.h>
#include <util/BoostVariantUtil.h>

#include <boost/variant.hpp>
#include <boost/utility/enable_if.hpp>
#include <boost/type_traits.hpp>
#include <boost/mpl/bool.hpp>
#include <boost/mpl/not.hpp>
#include <boost/bind.hpp>
#include <functional>

#include <algorithm>
#include <string>

NS_IZENELIB_IR_BEGIN

namespace indexmanager
{

template<class T>
struct IndexKeyType
{
    collectionid_t cid;
    fieldid_t fid;
    T value;

    IndexKeyType():cid(0),fid(0),value(T()) {}

    IndexKeyType(collectionid_t c, fieldid_t f, T& v)
            :cid(c), fid(f), value(v)
    {}
    template<class Archive> void serialize(Archive & ar,const unsigned int version)
    {
        ar & fid;
        ar & cid;
        ar & value;
    }

    int compare(const IndexKeyType<T>& other) const
    {
        if (cid != other.cid)
            return cid-other.cid;
        else
            if (fid != other.fid)
                return fid-other.fid;
            else
                return __compare(other,static_cast<boost::is_arithmetic<T>*>(0));
    }

    bool isPrefix(const IndexKeyType<T>& other) const
    {
        if (cid != other.cid)
            return false;
        else
        {
            if (fid != other.fid)
                return false;
            else
            {
                String str;
                other.value.substr(str, 0, value.length());
                if (str == value)
                    return true;
                else
                    return false;
            }
        }
    }

    void display() const
    {
        cout<<cid<<" ";
        cout<<fid<<" ";
        //cout<<value;
    }
    bool sameField(const IndexKeyType<T>& other) const
    {
        return ((cid == other.cid) && (fid == other.fid));
    }

private:
    int __compare(const IndexKeyType<T>& other, const boost::mpl::true_*) const
    {
        if ( value> other.value)
        {
            return 1;
        }
        else if (value < other.value)
        {
            return -1;
        }
        else
            return 0;
        //return (int)(value - other.value);
    }
    int __compare(const IndexKeyType<T>& other, const boost::mpl::false_*) const
    {
        return value.compare(other.value);
    }
};

/**
 * BTreeIndex
 */

template <class KeyType>
class BTreeIndex:public izenelib::sdb::IndexSDB<KeyType,docid_t>
{
    typedef izenelib::sdb::iKeyType<KeyType> myKeyType;
    typedef std::vector<docid_t> myValueType;
    typedef DataType<myKeyType, myValueType> myDataType;
public:
    typedef typename IndexSDB<KeyType,docid_t>::IndexSDBCursor IndexSDBCursor;

public:

    BTreeIndex(string& fileName) :
            izenelib::sdb::IndexSDB<KeyType,docid_t>(fileName) //0, use default compare
    {
    }
    bool remove(const KeyType& key, docid_t docID);

    template<typename SDBCursor>
    bool search(const KeyType& key, SDBCursor& locn)
    {
        izenelib::sdb::iKeyType<KeyType> ikey(key, 0);
        return this->_sdb.search(ikey, locn);
    }

    template<typename SDBCursor>
    bool seq(SDBCursor& locn, KeyType& key, std::vector<docid_t>& value, ESeqDirection sdir)
    {
        izenelib::sdb::iKeyType<KeyType> ikey(key, 0);
        return this->_sdb.seq(locn, ikey, value, sdir);
    }

    bool get(const KeyType& key,BitVector& result)
    {
        myValueType vdat;
        unsigned int offset = 0;
        myKeyType firstKey(key, 0);
        if ( !this->_sdb.hasKey(firstKey) )
            return false;
        do
        {
            myKeyType ikey(key, offset++);
            if (this->_sdb.getValue(ikey, vdat))
                for (size_t i=0; i<vdat.size(); i++)
                    result.set(vdat[i]);
            else
                break;
        }
        while (true);
        return true;
    }

    bool get_without(const KeyType& key,BitVector& result)
    {
        myValueType vdat;
        unsigned int offset = 0;
        myKeyType firstKey(key, 0);
        if ( !this->_sdb.hasKey(firstKey) )
            return false;
        result.setAll();
        do
        {
            myKeyType ikey(key, offset++);
            if (this->_sdb.getValue(ikey, vdat))
                for (size_t i=0; i<vdat.size(); i++)
                    result.clear(vdat[i]);
            else
                break;
        }
        while (true);
        return true;
    }

    void getGreat(const KeyType& key, BitVector& result)
    {
        KeyType temp = getNext(key);
        myKeyType ikey(temp, 0);
        IndexSDBCursor locn;
        this->_sdb.search(ikey, locn);
        myDataType rec;
        while (this->_sdb.seq(locn, ESD_FORWARD) )
        {
            if (this->_sdb.get(locn, rec))
            {
                vector<docid_t> vdat = rec.get_value();
                for (size_t i=0; i<vdat.size(); i++)
                    result.set(vdat[i]);
            }
        }

    }

    void getGreatEqual(const KeyType& key, BitVector& result)
    {
        get(key, result);
        getGreat(key, result);
    }

    void getLess(const KeyType& key, BitVector& result)
    {
        KeyType temp = getPrev(key);
        myKeyType ikey(temp, 0);
        IndexSDBCursor locn;
        this->_sdb.search(ikey, locn);
        myDataType rec;
        while (this->_sdb.seq(locn, ESD_BACKWARD) )
        {
            if (this->_sdb.get(locn, rec) )
            {
                vector<docid_t> vdat = rec.get_value();
                for (size_t i=0; i<vdat.size(); i++)
                    result.set(vdat[i]);
            }
        }

    }

    void getLessEqual(const KeyType& key, BitVector& result)
    {
        get(key, result);
        getLess(key, result);
    }

    void getPrefix(const KeyType& key, BitVector& result)
    {
        myKeyType ikey(key, 0);
        myValueType ival;

        IndexSDBCursor locn=	this->_sdb.search(ikey);
        while(this->_sdb.get(locn, ikey, ival) )
        {
            if( isPrefix1(key, ikey.key) )
            {
                for (size_t i=0; i<ival.size(); i++)
                    result.set(ival[i]);
                this->_sdb.seq(locn);	   
            }
            else
                break;
        }
    }
};

template <class KeyType>
bool BTreeIndex<KeyType>::remove(const KeyType& key, docid_t docID)
{
    vector<docid_t> vDat;
    getValue(key, vDat);

    //if we can make sure there are no duplicate inserting items, e.g, documentID
    //can't be the same as the existing items in IndexSDB, i.e nodup is true.
    vector<docid_t>::iterator iter = find(vDat.begin(), vDat.end(), docID);
    if (iter != vDat.end() )
    {
        vDat.erase(iter);
{'fraction_non_alphanumeric': 0.12421660446447433, 'fraction_numerical': 0.0035912963875783395, 'mean_word_length': 1.8737353298259813, 'pattern_counts': {'":': 0, '<': 122, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2483280', 'n_tokens_mistral': 4793, 'n_tokens_neox': 4682, 'n_words': 1012}