<commit_before>/**
* @file        BitVector.h
* @author     <NAME>
* @version     SF1 v5.0
* @brief 
*/
#ifndef BITVECTOR_H
#define BITVECTOR_H

#include <ir/index_manager/utility/system.h>
#include <ir/index_manager/store/Directory.h>
#include <ir/index_manager/store/IndexInput.h>
#include <ir/index_manager/store/IndexOutput.h>

#include <am/bitmap/Ewah.h>

NS_IZENELIB_IR_BEGIN

namespace indexmanager{
using namespace izenelib::am;
/*
*@brief BitVector
* An implementation of bitmap
*/
class BitVector
{
public:
    BitVector()
        : bits_(0), size_(0), maxBytesNum_(MAX_BYTES_NUM_INIT)
    {
        bits_ = new unsigned char[maxBytesNum_];
        memset(bits_, 0 , maxBytesNum_);
    }

    BitVector(const BitVector& other)
        : bits_(0), size_(other.size_), maxBytesNum_(other.maxBytesNum_)
    {
        bits_ = new unsigned char[maxBytesNum_];
        for(size_t i = 0; i < maxBytesNum_; ++i )
            bits_[i] = other.bits_[i];
    }

    BitVector(size_t n)
        : bits_(0), size_(n), maxBytesNum_(getMaxBytesNum(size_))
    {
        bits_ = new unsigned char[maxBytesNum_];
        memset(bits_, 0 , maxBytesNum_);
    }

    ~BitVector()
    {
        delete[] bits_;
    }

public:
    void set(size_t bit) 
    {
        const size_t byte = bit >> 3;
        if(byte >= maxBytesNum_)
            grow(bit + 1);
        if(bit >= size_)
            size_ = bit + 1;

        bits_[byte] |= 1 << (bit & 7);
    }

    void clear(size_t bit) 
    {
        const size_t byte = bit >> 3;
        if(byte >= maxBytesNum_)
            grow(bit + 1);
        if(bit >= size_)
            size_ = bit + 1;

        bits_[byte] &= ~(1 << (bit & 7));
    }

    void clear()
    {
        memset(bits_, 0 , getBytesNum(size_));
    }

    void setAll()
    {
        if(size_ == 0)
            return;

        const size_t fullByteNum = getBytesNum(size_) - 1;
        memset(bits_, 0xFF, fullByteNum);

        const size_t endBit = size_ - 1;
        unsigned char endMask = (1 << ((endBit & 7) + 1)) - 1;
        bits_[endBit >> 3] |= endMask;
    }

    bool test(size_t bit) const
    {
        if(bit >= size_)
            return false;

        return bits_[bit >> 3] & (1 << (bit & 7));
    }

    void compressed(EWAHBoolArray<uint32_t>& compressedBitMap) const
    {
        const size_t byteNum = getBytesNum(size_);
        unsigned int wordinbytes = sizeof(uint32_t);
        for(size_t i = 0; i < byteNum/wordinbytes; i++)
        {
            uint32_t currentword(0);
            for(size_t j = 0; j < wordinbytes; j++)
            {
                currentword |= static_cast<uint32_t>(bits_[i * wordinbytes + j]) << (j * 8);
            }
            compressedBitMap.add(currentword);
        }
        unsigned int leftBytes = byteNum % wordinbytes;
        if ( leftBytes != 0 )
        {
            uint32_t lastWord(0);
            for (size_t k = 0 ; k < leftBytes; k++)
            {
                lastWord |= static_cast<uint32_t>(bits_[(byteNum / wordinbytes) * wordinbytes + k]) << (k * 8);
            }
            compressedBitMap.add(lastWord);
        }
    }<commit_msg>add method to BitVector of importing dat from EWah<commit_after>/**
* @file        BitVector.h
* @author     <NAME>
* @version     SF1 v5.0
* @brief 
*/
#ifndef BITVECTOR_H
#define BITVECTOR_H

#include <ir/index_manager/utility/system.h>
#include <ir/index_manager/store/Directory.h>
#include <ir/index_manager/store/IndexInput.h>
#include <ir/index_manager/store/IndexOutput.h>

#include <am/bitmap/Ewah.h>

NS_IZENELIB_IR_BEGIN

namespace indexmanager{
using namespace izenelib::am;
/*
*@brief BitVector
* An implementation of bitmap
*/
class BitVector
{
public:
    BitVector()
        : bits_(0), size_(0), maxBytesNum_(MAX_BYTES_NUM_INIT)
    {
        bits_ = new unsigned char[maxBytesNum_];
        memset(bits_, 0 , maxBytesNum_);
    }

    BitVector(const BitVector& other)
        : bits_(0), size_(other.size_), maxBytesNum_(other.maxBytesNum_)
    {
        bits_ = new unsigned char[maxBytesNum_];
        for(size_t i = 0; i < maxBytesNum_; ++i )
            bits_[i] = other.bits_[i];
    }

    BitVector(size_t n)
        : bits_(0), size_(n), maxBytesNum_(getMaxBytesNum(size_))
    {
        bits_ = new unsigned char[maxBytesNum_];
        memset(bits_, 0 , maxBytesNum_);
    }

    ~BitVector()
    {
        delete[] bits_;
    }

public:
    void set(size_t bit) 
    {
        const size_t byte = bit >> 3;
        if(byte >= maxBytesNum_)
            grow(bit + 1);
        if(bit >= size_)
            size_ = bit + 1;

        bits_[byte] |= 1 << (bit & 7);
    }

    void clear(size_t bit) 
    {
        const size_t byte = bit >> 3;
        if(byte >= maxBytesNum_)
            grow(bit + 1);
        if(bit >= size_)
            size_ = bit + 1;

        bits_[byte] &= ~(1 << (bit & 7));
    }

    void clear()
    {
        memset(bits_, 0 , getBytesNum(size_));
    }

    void setAll()
    {
        if(size_ == 0)
            return;

        const size_t fullByteNum = getBytesNum(size_) - 1;
        memset(bits_, 0xFF, fullByteNum);

        const size_t endBit = size_ - 1;
        unsigned char endMask = (1 << ((endBit & 7) + 1)) - 1;
        bits_[endBit >> 3] |= endMask;
    }

    bool test(size_t bit) const
    {
        if(bit >= size_)
            return false;

        return bits_[bit >> 3] & (1 << (bit & 7));
    }

    void compressed(EWAHBoolArray<uint32_t>& compressedBitMap) const
    {
        const size_t byteNum = getBytesNum(size_);
        unsigned int wordinbytes = sizeof(uint32_t);
        for(size_t i = 0; i < byteNum/wordinbytes; i++)
        {
            uint32_t currentword(0);
            for(size_t j = 0; j < wordinbytes; j++)
            {
                currentword |= static_cast<uint32_t>(bits_[i * wordinbytes + j]) << (j * 8);
            }
            compressedBitMap.add(currentword);
        }
        unsigned int leftBytes = byteNum % wordinbytes;
        if ( leftBytes != 0 )
        {
            uint32_t lastWord(0);
            for (size_t k = 0 ; k < leftBytes; k++)
            {
                lastWord |= static_cast<uint32_t>(bits_[(byteNum / wordinbytes) * wordinbytes + k]) << (k * 8);
            }
            compressedBitMap.add(lastWord);
        }
    }

    void importFromEWAH(const EWAHBoolArray<uint32_t>& ewahBitmap)
    {
        uint pointer(0);
        uint currentoffset(0);
        const vector<uint32_t>& buffer = ewahBitmap.getBuffer();
        while (pointer <buffer.size())
        {
            ConstRunningLengthWord<uint32_t> rlw(buffer[pointer]);
            if (rlw.getRunningBit())
            {
                for (uint x = 0; x<  static_cast<uint>(rlw.getRunningLength()*32);++x)
                {
                set(currentoffset + x);
                }
            }
            currentoffset+=rlw.getRunningLength()*32;
            ++pointer;
            for (uint k = 0; k<rlw.getNumberOfLiteralWords();++k)
            {
                const uint32_t currentword = buffer[pointer];
                for (uint k = 0; k < 32; ++k)
                {
                    if ( (currentword & (static_cast<uint32_t>(1) << k)) != 0)
                        set(currentoffset + k);
                }
                currentoffset+=32;
                ++pointer;
            }
        }
    }
{'fraction_non_alphanumeric': 0.12781852757402878, 'fraction_numerical': 0.017794077696278186, 'mean_word_length': 1.8527702440914375, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3390662', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2409, 'n_words': 524}