<commit_before>#include "FileBuffer.h"

FileView::FileView(QString &path, bufsize_t maxSize)
    : fIn (path)
{
    if (fIn.open(QFile::ReadOnly | QFile::Truncate) == false) {
        throw FileBufferException("Cannot open the file: " + path);
    }
    this->fileSize = fIn.size();
    //if (DBG_LVL) printf("File of size:\t%lld\n", fileSize);
    bufsize_t readableSize = getMappableSize(fIn);
    this->mappedSize = (readableSize > maxSize) ? maxSize : readableSize;
    //printf("Mapping size: %lx = %ld\n", this->mappedSize, this->mappedSize);
    uchar *pData = fIn.map(0, this->mappedSize);
    if (pData == NULL) {
        throw BufferException("Cannot map the file: " + path + " of size: 0x" + QString::number(this->mappedSize, 16));
    }
    this->mappedContent = (BYTE*) pData;
}

FileView::~FileView()
{
    fIn.unmap((uchar*)this->mappedContent);
    fIn.close();
}

bufsize_t FileView::getMappableSize(QFile &fIn)
{
    bufsize_t size = getReadableSize(fIn);
    if (size > FILEVIEW_MAXSIZE) {
        size = FILEVIEW_MAXSIZE;
    }
    return size;
}
//----------------------------------------------------------------

ByteBuffer* FileBuffer::read(QString &path, bufsize_t minBufSize)
{
    //printf("reading file...");
    QFile fIn(path);
    if (fIn.open(QFile::ReadOnly | QFile::Truncate) == false) {
        throw FileBufferException("Cannot open the file: " + path);
    }
    bufsize_t readableSize = getReadableSize(fIn);
    bufsize_t allocSize = (readableSize < minBufSize) ? minBufSize : readableSize;
    //printf("Alloc size: %lx\n", allocSize);

    ByteBuffer *bufferedFile = new ByteBuffer(allocSize); //throws exceptions
    char *content = (char*) bufferedFile->getContent();
    bufsize_t contentSize = bufferedFile->getContentSize();

    if (content == NULL) throw FileBufferException("Cannot allocate buffer");
    //printf("Reading...%lx , BUFSIZE_MAX = %lx\n", allocSize, BUFSIZE_MAX);

    readableSize = contentSize;

    bufsize_t redSize = 0;
    offset_t prevOffset = 0;
    offset_t maxOffset = readableSize - 1;

    while (fIn.pos() < maxOffset) {
        redSize += fIn.read(content + redSize,  FILEVIEW_MAXSIZE);
        if (prevOffset == fIn.pos()) break; //cannot read more!
        printf("pos = %llx redSize = %llx\n", fIn.pos(), redSize);
        prevOffset = fIn.pos();
    }
    //printf("Red size...%lx\n", redSize);
    
    fIn.close();
    return bufferedFile;
}

bufsize_t FileBuffer::getReadableSize(QFile &fIn)
{
    qint64 fileSize = fIn.size();
    bufsize_t size = static_cast<bufsize_t> (fileSize);

    if (size > FILE_MAXSIZE) {<commit_msg>[W-I-P] In FileBuffer::read - fixed adjusting max size<commit_after>#include "FileBuffer.h"

FileView::FileView(QString &path, bufsize_t maxSize)
    : fIn (path)
{
    if (fIn.open(QFile::ReadOnly | QFile::Truncate) == false) {
        throw FileBufferException("Cannot open the file: " + path);
    }
    this->fileSize = fIn.size();
    //if (DBG_LVL) printf("File of size:\t%lld\n", fileSize);
    bufsize_t readableSize = getMappableSize(fIn);
    this->mappedSize = (readableSize > maxSize) ? maxSize : readableSize;
    //printf("Mapping size: %lx = %ld\n", this->mappedSize, this->mappedSize);
    uchar *pData = fIn.map(0, this->mappedSize);
    if (pData == NULL) {
        throw BufferException("Cannot map the file: " + path + " of size: 0x" + QString::number(this->mappedSize, 16));
    }
    this->mappedContent = (BYTE*) pData;
}

FileView::~FileView()
{
    fIn.unmap((uchar*)this->mappedContent);
    fIn.close();
}

bufsize_t FileView::getMappableSize(QFile &fIn)
{
    bufsize_t size = getReadableSize(fIn);
    if (size > FILEVIEW_MAXSIZE) {
        size = FILEVIEW_MAXSIZE;
    }
    return size;
}
//----------------------------------------------------------------

ByteBuffer* FileBuffer::read(QString &path, bufsize_t minBufSize)
{
    //printf("reading file...");
    QFile fIn(path);
    if (fIn.open(QFile::ReadOnly | QFile::Truncate) == false) {
        throw FileBufferException("Cannot open the file: " + path);
    }
    bufsize_t readableSize = getReadableSize(fIn);
    bufsize_t allocSize = (readableSize < minBufSize) ? minBufSize : readableSize;
    //printf("Alloc size: %lx\n", allocSize);

    ByteBuffer *bufferedFile = new ByteBuffer(allocSize); //throws exceptions
    char *content = (char*) bufferedFile->getContent();
    bufsize_t contentSize = bufferedFile->getContentSize();

    if (content == NULL) throw FileBufferException("Cannot allocate buffer");
    //printf("Reading...%lx , BUFSIZE_MAX = %lx\n", allocSize, BUFSIZE_MAX);

    bufsize_t redSize = 0;
    offset_t prevOffset = 0;
    offset_t maxOffset = contentSize - 1;

    while (fIn.pos() < maxOffset) {

        bufsize_t maxSize = contentSize - redSize;
        if (maxSize > FILEVIEW_MAXSIZE) maxSize = FILEVIEW_MAXSIZE;

        redSize += fIn.read(content + redSize,  maxSize);
        if (prevOffset == fIn.pos()) break; //cannot read more!
        printf("redSize = %lx\n", redSize);
        prevOffset = fIn.pos();
    }
    //printf("Red size...%lx\n", redSize);
    
    fIn.close();
    return bufferedFile;
}

bufsize_t FileBuffer::getReadableSize(QFile &fIn)
{
    qint64 fileSize = fIn.size();
    bufsize_t size = static_cast<bufsize_t> (fileSize);

    if (size > FILE_MAXSIZE) {
{'fraction_non_alphanumeric': 0.16911211437170806, 'fraction_numerical': 0.003386004514672686, 'mean_word_length': 3.108964451313756, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6161949', 'n_tokens_mistral': 1837, 'n_tokens_neox': 1772, 'n_words': 386}