<commit_before>#include "libpstack/inflatereader.h"
#include "libpstack/util.h"

#include <zlib.h>

InflateReader::InflateReader(size_t inflatedSize, const Reader &upstream)
    : MemReader(std::string("lzma-decoded content from ") + stringify(upstream), inflatedSize, new char[inflatedSize])
{
    char xferbuf[32768];

    z_stream stream{};

    int window = 15;
    if (inflateInit2(&stream, window) != Z_OK)
        throw (Exception() << "inflateInit2 failed");

    stream.avail_out = inflatedSize;
    using bytep = Bytef *;
    stream.next_out = bytep(data);
    bool eof = false;
    size_t inputOffset = 0;
    if (verbose >= 2)
        *debug << "inflating" << upstream << "...";
    for (bool done = false; !done; ) {
        if (stream.avail_in == 0 && !eof) {
            // keep the input buffer full
            stream.avail_in = upstream.read(inputOffset, sizeof xferbuf, xferbuf);
            inputOffset += stream.avail_in;
            stream.next_in = bytep(xferbuf);
            if (stream.avail_in == 0)
                eof = true;
        }
        size_t writeChunk = stream.avail_out;
        switch (inflate(&stream, eof ? Z_FINISH : Z_SYNC_FLUSH)) {
            case Z_STREAM_END:
                done = true;
                // fallthrough
            case Z_OK:<commit_msg>fix description: "inflate" is not lzma<commit_after>#include "libpstack/inflatereader.h"
#include "libpstack/util.h"

#include <zlib.h>

InflateReader::InflateReader(size_t inflatedSize, const Reader &upstream)
    : MemReader(std::string("inflated content from ") + stringify(upstream), inflatedSize, new char[inflatedSize])
{
    char xferbuf[32768];

    z_stream stream{};

    int window = 15;
    if (inflateInit2(&stream, window) != Z_OK)
        throw (Exception() << "inflateInit2 failed");

    stream.avail_out = inflatedSize;
    using bytep = Bytef *;
    stream.next_out = bytep(data);
    bool eof = false;
    size_t inputOffset = 0;
    if (verbose >= 2)
        *debug << "inflating" << upstream << "...";
    for (bool done = false; !done; ) {
        if (stream.avail_in == 0 && !eof) {
            // keep the input buffer full
            stream.avail_in = upstream.read(inputOffset, sizeof xferbuf, xferbuf);
            inputOffset += stream.avail_in;
            stream.next_in = bytep(xferbuf);
            if (stream.avail_in == 0)
                eof = true;
        }
        size_t writeChunk = stream.avail_out;
        switch (inflate(&stream, eof ? Z_FINISH : Z_SYNC_FLUSH)) {
            case Z_STREAM_END:
                done = true;
                // fallthrough
            case Z_OK:
{'fraction_non_alphanumeric': 0.12342857142857143, 'fraction_numerical': 0.009904761904761904, 'mean_word_length': 2.3796653796653797, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '441414', 'n_tokens_mistral': 892, 'n_tokens_neox': 843, 'n_words': 194}