<commit_before>#pragma once

#include "Event.h"
#include "FileContentSearchData.h"
#include "HandleHolder.h"
#include "HeapArray.h"
#include "ObjectPool.h"
#include "ReadBatch.h"
#include "WorkQueue.h"

struct SearchInstructions;
class SearchResultReporter;
class StringSearcher;

class FileReadWorkQueue : WorkQueue<FileReadData>, ThreadedWorkQueue<FileReadWorkQueue, SlotSearchData>
{
public:
    FileReadWorkQueue(const StringSearcher& stringSearcher, const SearchInstructions& searchInstructions, SearchResultReporter& searchResultReporter);
    ~FileReadWorkQueue();

    void Initialize();
    void DrainWorkQueue();
    void CompleteAllWork();

    static constexpr uint16_t kFileReadSlotCount = DSTORAGE_MAX_QUEUE_CAPACITY;
    static constexpr size_t kFileReadBufferBaseSize = 512 * 1024 * 1024 / kFileReadSlotCount; // 512 MB total

private:
    typedef WorkQueue<FileReadData> MyFileReadBase;
    typedef ThreadedWorkQueue<FileReadWorkQueue, SlotSearchData> MySearchResultBase;
    friend class MyFileReadBase;<commit_msg>Adjusting buffer size for more balanced performance.<commit_after>#pragma once

#include "Event.h"
#include "FileContentSearchData.h"
#include "HandleHolder.h"
#include "HeapArray.h"
#include "ObjectPool.h"
#include "ReadBatch.h"
#include "WorkQueue.h"

struct SearchInstructions;
class SearchResultReporter;
class StringSearcher;

class FileReadWorkQueue : WorkQueue<FileReadData>, ThreadedWorkQueue<FileReadWorkQueue, SlotSearchData>
{
public:
    FileReadWorkQueue(const StringSearcher& stringSearcher, const SearchInstructions& searchInstructions, SearchResultReporter& searchResultReporter);
    ~FileReadWorkQueue();

    void Initialize();
    void DrainWorkQueue();
    void CompleteAllWork();

    static constexpr size_t kTargetTotalBufferSize = 512 * 1024 * 1024; // 512 MB total
    static constexpr size_t kFileReadBufferBaseSize = 128 * 1024;
    static constexpr uint16_t kFileReadSlotCount = kTargetTotalBufferSize / kFileReadBufferBaseSize;

private:
    typedef WorkQueue<FileReadData> MyFileReadBase;
    typedef ThreadedWorkQueue<FileReadWorkQueue, SlotSearchData> MySearchResultBase;
    friend class MyFileReadBase;
{'fraction_non_alphanumeric': 0.07834101382488479, 'fraction_numerical': 0.017972350230414748, 'mean_word_length': 6.590909090909091, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5877663', 'n_tokens_mistral': 670, 'n_tokens_neox': 628, 'n_words': 151}