<commit_before>#include <zlib.h>
#include <assert.h>
#include <string.h>

#include <algorithm>
#include <stdexcept>

#include "deflatedbuffer.h"
namespace avro {

DeflatedBuffer::DeflatedBuffer() {
    buf.resize(5 * 1024 * 1024);
}

void DeflatedBuffer::assignData(const char *compressedData, size_t compressedDataLength) {
    length = 0;
    pointer = 0;
    documentStartPointer = 0;

    decompress(compressedData, compressedDataLength);
}

void DeflatedBuffer::decompress(const char *compressedData, size_t compressedDataLength) {

    int ret;
    z_stream strm;

    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.avail_in = 0;
    strm.next_in = Z_NULL;
    ret = inflateInit2(&strm, -15);
    assert(ret == Z_OK);

    size_t avail = buf.size();
    size_t offset = 0;

    /* decompress until deflate stream ends or end of file */
    do {
        strm.avail_in = avail;

        if (strm.avail_in == 0)
            break;
        strm.next_in = (unsigned char *)(compressedData + offset);
        do {

            strm.avail_out = (uInt)buf.size() - offset;
            strm.next_out = buf.data() + offset;
            ret = inflate(&strm, Z_NO_FLUSH);
            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
            switch (ret) {
            case Z_NEED_DICT:
                // ret = Z_DATA_ERROR;     /* and fall through */
            case Z_DATA_ERROR:
            case Z_MEM_ERROR:
                (void)inflateEnd(&strm);
                throw std::runtime_error("Z_MEM_ERROR/Z_DATA_ERROR/Z_NEED_DICT");
            }

        } while (strm.avail_out == 0);

        avail -= strm.avail_in;
        offset += strm.avail_in;

        /* done when inflate() says it's done */
    } while (ret != Z_STREAM_END);

    length = strm.total_out;
}

char DeflatedBuffer::getChar() {
    char result = buf[pointer];
    pointer++;
    return result;
}

void DeflatedBuffer::read(void *to, size_t len) {
    memcpy(to, buf.data() + pointer, len);
    pointer += len;
}

size_t DeflatedBuffer::size() {
    return length;
}

void DeflatedBuffer::startDocument() {
    documentStartPointer = pointer;
}

void DeflatedBuffer::resetToDocument() {
    pointer = documentStartPointer;
}

std::string DeflatedBuffer::getStdString(size_t len) {
    std::string result;
    result.resize(len);

    read((void*)result.data(), len);

    return result;
}

StringBuffer DeflatedBuffer::getString(size_t len) {
    StringBuffer result((const char*)buf.data() + pointer, len);
    pointer += len;
    return result;
}

}
<commit_msg>Finish up with libz when unpack is done<commit_after>#include <zlib.h>
#include <assert.h>
#include <string.h>

#include <algorithm>
#include <stdexcept>

#include "deflatedbuffer.h"
namespace avro {

DeflatedBuffer::DeflatedBuffer() {
    buf.resize(5 * 1024 * 1024);
}

void DeflatedBuffer::assignData(const char *compressedData, size_t compressedDataLength) {
    length = 0;
    pointer = 0;
    documentStartPointer = 0;

    decompress(compressedData, compressedDataLength);
}

void DeflatedBuffer::decompress(const char *compressedData, size_t compressedDataLength) {

    int ret;
    z_stream strm;

    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.avail_in = 0;
    strm.next_in = Z_NULL;
    ret = inflateInit2(&strm, -15);
    assert(ret == Z_OK);

    size_t avail = buf.size();
    size_t offset = 0;

    /* decompress until deflate stream ends or end of file */
    do {
        strm.avail_in = avail;

        if (strm.avail_in == 0)
            break;
        strm.next_in = (unsigned char *)(compressedData + offset);
        do {

            strm.avail_out = (uInt)buf.size() - offset;
            strm.next_out = buf.data() + offset;
            ret = inflate(&strm, Z_NO_FLUSH);
            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
            switch (ret) {
            case Z_NEED_DICT:
                // ret = Z_DATA_ERROR;     /* and fall through */
            case Z_DATA_ERROR:
            case Z_MEM_ERROR:
                (void)inflateEnd(&strm);
                throw std::runtime_error("Z_MEM_ERROR/Z_DATA_ERROR/Z_NEED_DICT");
            }

        } while (strm.avail_out == 0);

        avail -= strm.avail_in;
        offset += strm.avail_in;

        /* done when inflate() says it's done */
    } while (ret != Z_STREAM_END);

    length = strm.total_out;
    inflateEnd(&strm);
}

char DeflatedBuffer::getChar() {
    char result = buf[pointer];
    pointer++;
    return result;
}

void DeflatedBuffer::read(void *to, size_t len) {
    memcpy(to, buf.data() + pointer, len);
    pointer += len;
}

size_t DeflatedBuffer::size() {
    return length;
}

void DeflatedBuffer::startDocument() {
    documentStartPointer = pointer;
}

void DeflatedBuffer::resetToDocument() {
    pointer = documentStartPointer;
}

std::string DeflatedBuffer::getStdString(size_t len) {
    std::string result;
    result.resize(len);

    read((void*)result.data(), len);

    return result;
}

StringBuffer DeflatedBuffer::getString(size_t len) {
    StringBuffer result((const char*)buf.data() + pointer, len);
    pointer += len;
    return result;
}

}

{'fraction_non_alphanumeric': 0.12382595361318766, 'fraction_numerical': 0.007283879624305156, 'mean_word_length': 2.6720619282195637, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7529637', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1727, 'n_words': 380}