<commit_before>#include "pch.h"
#include "memory_stream.h"




MemoryStream::MemoryStream()
{
}

MemoryStream::~MemoryStream()
{
    close();
}

void MemoryStream::close()
{
    _ownedBuffer.reset();
    _readBuffer = nullptr;
    _writeBuffer = nullptr;
    _cursor = _bufferSize = 0;
    _autoBuffer.clear();
}

MemoryStream * MemoryStream::create(const void * buffer, size_t bufferSize)
{
    if (!buffer)
        return nullptr;

    MemoryStream * res = new MemoryStream();
    res->_readBuffer = reinterpret_cast<const uint8_t *>(buffer);
    res->_writeBuffer = nullptr;
    res->_bufferSize = bufferSize;
    res->_cursor = 0;
    res->_canAllocate = false;

    return res;
}

MemoryStream * MemoryStream::create(void * buffer, size_t bufferSize)
{
    if (!buffer)
        return nullptr;

    MemoryStream * res = new MemoryStream();
    res->_readBuffer = reinterpret_cast<const uint8_t *>(buffer);
    res->_writeBuffer = reinterpret_cast<uint8_t *>(buffer);
    res->_bufferSize = bufferSize;
    res->_cursor = 0;
    res->_canAllocate = false;

    return res;
}

MemoryStream * MemoryStream::create(std::unique_ptr< uint8_t[] >& buffer, size_t bufferSize)
{
    if (!buffer.get())
        return nullptr;

    MemoryStream * res = new MemoryStream();
    res->_ownedBuffer.reset(buffer.release());
    res->_readBuffer = reinterpret_cast<const uint8_t *>(res->_ownedBuffer.get());
    res->_writeBuffer = reinterpret_cast<uint8_t *>(res->_ownedBuffer.get());
    res->_bufferSize = bufferSize;
    res->_cursor = 0;
    res->_canAllocate = false;

    return res;
}

MemoryStream * MemoryStream::create()
{
    MemoryStream * res = new MemoryStream();
    res->_readBuffer = nullptr;
    res->_writeBuffer = nullptr;
    res->_bufferSize = 0;
    res->_cursor = 0;
    res->_canAllocate = true;

    return res;
}

size_t MemoryStream::read(void* ptr, size_t size, size_t count)
{
    if (size == 0)
        return 0;

    size_t maxReadBytes = std::min(_bufferSize - _cursor, size * count);
    size_t maxReadElements = maxReadBytes / size;

    if (maxReadElements == 0)
        return 0;

    maxReadBytes = maxReadElements * size;
    memcpy(ptr, _readBuffer + _cursor, maxReadBytes);
    _cursor += maxReadBytes;
    return maxReadElements;
}

char* MemoryStream::readLine(char* str, int num)
{
    if (num <= 0)
        return NULL;

    size_t maxReadBytes = std::min<size_t>(_bufferSize - _cursor, num);
    char * strSave = str;
    while (maxReadBytes > 0)
    {
        const char& ch = *(_readBuffer + _cursor);
        if (ch == '\r' || ch == '\n')
        {
            *str++ = ch;
            if (str - strSave < num)
                *str++ = '\0';
            return strSave;
        }
<commit_msg>fixed readLine method of MemoryStream<commit_after>#include "pch.h"
#include "memory_stream.h"




MemoryStream::MemoryStream()
{
}

MemoryStream::~MemoryStream()
{
    close();
}

void MemoryStream::close()
{
    _ownedBuffer.reset();
    _readBuffer = nullptr;
    _writeBuffer = nullptr;
    _cursor = _bufferSize = 0;
    _autoBuffer.clear();
}

MemoryStream * MemoryStream::create(const void * buffer, size_t bufferSize)
{
    if (!buffer)
        return nullptr;

    MemoryStream * res = new MemoryStream();
    res->_readBuffer = reinterpret_cast<const uint8_t *>(buffer);
    res->_writeBuffer = nullptr;
    res->_bufferSize = bufferSize;
    res->_cursor = 0;
    res->_canAllocate = false;

    return res;
}

MemoryStream * MemoryStream::create(void * buffer, size_t bufferSize)
{
    if (!buffer)
        return nullptr;

    MemoryStream * res = new MemoryStream();
    res->_readBuffer = reinterpret_cast<const uint8_t *>(buffer);
    res->_writeBuffer = reinterpret_cast<uint8_t *>(buffer);
    res->_bufferSize = bufferSize;
    res->_cursor = 0;
    res->_canAllocate = false;

    return res;
}

MemoryStream * MemoryStream::create(std::unique_ptr< uint8_t[] >& buffer, size_t bufferSize)
{
    if (!buffer.get())
        return nullptr;

    MemoryStream * res = new MemoryStream();
    res->_ownedBuffer.reset(buffer.release());
    res->_readBuffer = reinterpret_cast<const uint8_t *>(res->_ownedBuffer.get());
    res->_writeBuffer = reinterpret_cast<uint8_t *>(res->_ownedBuffer.get());
    res->_bufferSize = bufferSize;
    res->_cursor = 0;
    res->_canAllocate = false;

    return res;
}

MemoryStream * MemoryStream::create()
{
    MemoryStream * res = new MemoryStream();
    res->_readBuffer = nullptr;
    res->_writeBuffer = nullptr;
    res->_bufferSize = 0;
    res->_cursor = 0;
    res->_canAllocate = true;

    return res;
}

size_t MemoryStream::read(void* ptr, size_t size, size_t count)
{
    if (size == 0)
        return 0;

    size_t maxReadBytes = std::min(_bufferSize - _cursor, size * count);
    size_t maxReadElements = maxReadBytes / size;

    if (maxReadElements == 0)
        return 0;

    maxReadBytes = maxReadElements * size;
    memcpy(ptr, _readBuffer + _cursor, maxReadBytes);
    _cursor += maxReadBytes;
    return maxReadElements;
}

char* MemoryStream::readLine(char* str, int num)
{
    if (num <= 0)
        return NULL;

    size_t maxReadBytes = std::min<size_t>(_bufferSize - _cursor, num);
    char * strSave = str;
    while (maxReadBytes > 0)
    {
        const char& ch = *(_readBuffer + _cursor);
        if (ch == '\r' || ch == '\n')
        {
            *str++ = ch;
            _cursor++;
            if (str - strSave < num)
                *str++ = '\0';
            return strSave;
        }

{'fraction_non_alphanumeric': 0.13336962438758845, 'fraction_numerical': 0.006895300308473961, 'mean_word_length': 2.976911976911977, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1201924', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1733, 'n_words': 407}