<commit_before>#include "StorageChunkReader.h"
#include "System/Time.h"

void StorageChunkReader::Open(
 ReadBuffer filename, uint64_t preloadThreshold_,
 bool keysOnly_, bool forwardDirection_)
{
    count = 0;
    numRead = 0;

    forwardDirection = forwardDirection_;
    preloadThreshold = preloadThreshold_;
    keysOnly = keysOnly_;

    fileChunk.useCache = false;
    fileChunk.SetFilename(filename);
    fileChunk.ReadHeaderPage();
    fileChunk.LoadIndexPage();
}

void StorageChunkReader::SetEndKey(ReadBuffer endKey_)
{
    endKey = endKey;
}

void StorageChunkReader::SetPrefix(ReadBuffer prefix_)
{
    prefix = prefix_;
}

void StorageChunkReader::SetCount(unsigned count_)
{
    count = count_;
}

StorageFileKeyValue* StorageChunkReader::First(ReadBuffer& firstKey)
{
    bool                    ret;
    int                     cmpres;
    StorageFileKeyValue*    it;
    
    prevIndex = 0;

    if (forwardDirection && ReadBuffer::Cmp(firstKey, fileChunk.indexPage->GetFirstKey()) < 0)
    {
        index = 0;
        offset = fileChunk.indexPage->GetFirstDatapageOffset();
    }
    else if (!forwardDirection && firstKey.GetLength() > 0 && ReadBuffer::Cmp(firstKey, fileChunk.indexPage->GetFirstKey()) < 0)<commit_msg>Fixed typo.<commit_after>#include "StorageChunkReader.h"
#include "System/Time.h"

void StorageChunkReader::Open(
 ReadBuffer filename, uint64_t preloadThreshold_,
 bool keysOnly_, bool forwardDirection_)
{
    count = 0;
    numRead = 0;

    forwardDirection = forwardDirection_;
    preloadThreshold = preloadThreshold_;
    keysOnly = keysOnly_;

    fileChunk.useCache = false;
    fileChunk.SetFilename(filename);
    fileChunk.ReadHeaderPage();
    fileChunk.LoadIndexPage();
}

void StorageChunkReader::SetEndKey(ReadBuffer endKey_)
{
    endKey = endKey_;
}

void StorageChunkReader::SetPrefix(ReadBuffer prefix_)
{
    prefix = prefix_;
}

void StorageChunkReader::SetCount(unsigned count_)
{
    count = count_;
}

StorageFileKeyValue* StorageChunkReader::First(ReadBuffer& firstKey)
{
    bool                    ret;
    int                     cmpres;
    StorageFileKeyValue*    it;
    
    prevIndex = 0;

    if (forwardDirection && ReadBuffer::Cmp(firstKey, fileChunk.indexPage->GetFirstKey()) < 0)
    {
        index = 0;
        offset = fileChunk.indexPage->GetFirstDatapageOffset();
    }
    else if (!forwardDirection && firstKey.GetLength() > 0 && ReadBuffer::Cmp(firstKey, fileChunk.indexPage->GetFirstKey()) < 0)
{'fraction_non_alphanumeric': 0.10592025775271849, 'fraction_numerical': 0.00724929520741039, 'mean_word_length': 3.625698324022346, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5098094', 'n_tokens_mistral': 759, 'n_tokens_neox': 729, 'n_words': 144}