<commit_before>#include "StorageDataPage.h"
#include "StorageFileChunk.h"

static int KeyCmp(const ReadBuffer& a, const ReadBuffer& b)
{
    return ReadBuffer::Cmp(a, b);
}

static const ReadBuffer Key(StorageFileKeyValue* kv)
{
    return kv->GetKey();
}

StorageDataPage::StorageDataPage(StorageFileChunk* owner_, uint32_t index_)
{
    size = 0;

    buffer.Allocate(STORAGE_DEFAULT_PAGE_GRAN);
    buffer.Zero();

    buffer.AppendLittle32(0); // dummy for size
    buffer.AppendLittle32(0); // dummy for checksum
    buffer.AppendLittle32(0); // dummy for numKeys
    
    owner = owner_;
    index = index_;
}

StorageDataPage::~StorageDataPage()
{
    keyValues.DeleteTree();
}

uint32_t StorageDataPage::GetSize()
{
    return size;
}

StorageKeyValue* StorageDataPage::Get(ReadBuffer& key)
{
    StorageFileKeyValue* it;

    int cmpres;
    
    if (GetNumKeys() == 0)
        return NULL;
        
    it = keyValues.Locate<ReadBuffer&>(key, cmpres);
//    it = LocateKeyValue(key, cmpres);
    if (cmpres != 0)
        return NULL;

    return it;
}

//StorageFileKeyValue* StorageDataPage::LocateKeyValue(ReadBuffer& key, int& cmpres)
//{
//    StorageFileKeyValue**   kvIndex;
//    unsigned                num;
//    unsigned                dist;
//    int                     curr;
//    
//    kvIndex = (StorageFileKeyValue**) keyValueIndexBuffer.GetBuffer();
//    num = keyValueIndexBuffer.GetLength() / sizeof(StorageFileKeyValue*);
//    
//    dist = num / 2;
//    curr = dist;
//    while (true)
//    {
//        cmpres = ReadBuffer::Cmp(key, kvIndex[curr]->GetKey());
//        if (cmpres == 0)
//            return kvIndex[curr];
//        
//        dist = num / 2;
//        if (dist == 0)
//            return NULL;
//        
//        if (cmpres < 0)
//        {
//            curr -= dist;
//            if (curr < 0)
//                curr = 0;
//        }
//        else 
//        {
//            curr += dist;
//            if (curr >= (int) num)
//                curr = num;
//        }
//    }
//}

uint32_t StorageDataPage::GetNumKeys()
{
    return keyValues.GetCount();
//    return keyValueIndexBuffer.GetLength() / sizeof(StorageFileKeyValue*);
}

uint32_t StorageDataPage::GetLength()
{
    return buffer.GetLength();
}

uint32_t StorageDataPage::GetIncrement(StorageKeyValue* kv)
{
    if (kv->GetType() == STORAGE_KEYVALUE_TYPE_SET)
        return (1 + 2 + kv->GetKey().GetLength() + 4 + kv->GetValue().GetLength());
    else if (kv->GetType() == STORAGE_KEYVALUE_TYPE_DELETE)
        return (1 + 2 + kv->GetKey().GetLength());
    else
        ASSERT_FAIL();

    return 0;
}

void StorageDataPage::Append(StorageKeyValue* kv)
{
    StorageFileKeyValue*    fkv;
    
    buffer.Append(kv->GetType());                           // 1 byte(s)
    buffer.AppendLittle16(kv->GetKey().GetLength());        // 2 byte(s)
    buffer.Append(kv->GetKey());
    if (kv->GetType() == STORAGE_KEYVALUE_TYPE_SET)
    {
        buffer.AppendLittle32(kv->GetValue().GetLength());  // 4 bytes(s)
        buffer.Append(kv->GetValue());
    }

    fkv = new StorageFileKeyValue;
    if (kv->GetType() == STORAGE_KEYVALUE_TYPE_SET)
        fkv->Set(ReadBuffer(kv->GetKey()), ReadBuffer(kv->GetValue()));
    else
        fkv->Delete(ReadBuffer(kv->GetValue()));
    
    keyValues.Insert(fkv);
//    AppendKeyValue(fkv);
}

void StorageDataPage::Finalize()<commit_msg>Fixed bug.<commit_after>#include "StorageDataPage.h"
#include "StorageFileChunk.h"

static int KeyCmp(const ReadBuffer& a, const ReadBuffer& b)
{
    return ReadBuffer::Cmp(a, b);
}

static const ReadBuffer Key(StorageFileKeyValue* kv)
{
    return kv->GetKey();
}

StorageDataPage::StorageDataPage(StorageFileChunk* owner_, uint32_t index_)
{
    size = 0;

    buffer.Allocate(STORAGE_DEFAULT_PAGE_GRAN);
    buffer.Zero();

    buffer.AppendLittle32(0); // dummy for size
    buffer.AppendLittle32(0); // dummy for checksum
    buffer.AppendLittle32(0); // dummy for numKeys
    
    owner = owner_;
    index = index_;
}

StorageDataPage::~StorageDataPage()
{
    keyValues.DeleteTree();
}

uint32_t StorageDataPage::GetSize()
{
    return size;
}

StorageKeyValue* StorageDataPage::Get(ReadBuffer& key)
{
    StorageFileKeyValue* it;

    int cmpres;
    
    if (GetNumKeys() == 0)
        return NULL;
        
    it = keyValues.Locate<ReadBuffer&>(key, cmpres);
//    it = LocateKeyValue(key, cmpres);
    if (cmpres != 0)
        return NULL;

    return it;
}

//StorageFileKeyValue* StorageDataPage::LocateKeyValue(ReadBuffer& key, int& cmpres)
//{
//    StorageFileKeyValue**   kvIndex;
//    unsigned                num;
//    unsigned                dist;
//    int                     curr;
//    
//    kvIndex = (StorageFileKeyValue**) keyValueIndexBuffer.GetBuffer();
//    num = keyValueIndexBuffer.GetLength() / sizeof(StorageFileKeyValue*);
//    
//    dist = num / 2;
//    curr = dist;
//    while (true)
//    {
//        cmpres = ReadBuffer::Cmp(key, kvIndex[curr]->GetKey());
//        if (cmpres == 0)
//            return kvIndex[curr];
//        
//        dist = num / 2;
//        if (dist == 0)
//            return NULL;
//        
//        if (cmpres < 0)
//        {
//            curr -= dist;
//            if (curr < 0)
//                curr = 0;
//        }
//        else 
//        {
//            curr += dist;
//            if (curr >= (int) num)
//                curr = num;
//        }
//    }
//}

uint32_t StorageDataPage::GetNumKeys()
{
    return keyValues.GetCount();
//    return keyValueIndexBuffer.GetLength() / sizeof(StorageFileKeyValue*);
}

uint32_t StorageDataPage::GetLength()
{
    return buffer.GetLength();
}

uint32_t StorageDataPage::GetIncrement(StorageKeyValue* kv)
{
    if (kv->GetType() == STORAGE_KEYVALUE_TYPE_SET)
        return (1 + 2 + kv->GetKey().GetLength() + 4 + kv->GetValue().GetLength());
    else if (kv->GetType() == STORAGE_KEYVALUE_TYPE_DELETE)
        return (1 + 2 + kv->GetKey().GetLength());
    else
        ASSERT_FAIL();

    return 0;
}

void StorageDataPage::Append(StorageKeyValue* kv)
{
    StorageFileKeyValue*    fkv;
    
    buffer.Append(kv->GetType());                           // 1 byte(s)
    buffer.AppendLittle16(kv->GetKey().GetLength());        // 2 byte(s)
    buffer.Append(kv->GetKey());
    if (kv->GetType() == STORAGE_KEYVALUE_TYPE_SET)
    {
        buffer.AppendLittle32(kv->GetValue().GetLength());  // 4 bytes(s)
        buffer.Append(kv->GetValue());
    }

    fkv = new StorageFileKeyValue;
    if (kv->GetType() == STORAGE_KEYVALUE_TYPE_SET)
        fkv->Set(ReadBuffer(kv->GetKey()), ReadBuffer(kv->GetValue()));
    else
        fkv->Delete(ReadBuffer(kv->GetKey()));
    
    keyValues.Insert(fkv);
//    AppendKeyValue(fkv);
}

void StorageDataPage::Finalize()
{'fraction_non_alphanumeric': 0.15333627927529828, 'fraction_numerical': 0.012372956252761821, 'mean_word_length': 2.2786093674553354, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3820488', 'n_tokens_mistral': 2302, 'n_tokens_neox': 2215, 'n_words': 412}