<commit_before>/* Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file */

#include "lib.h"

#ifdef HAVE_LZMA

#include "istream-private.h"
#include "istream-zlib.h"
#include <lzma.h>

#define CHUNK_SIZE (1024*64)

#define LZMA_MEMORY_LIMIT (1024*1024*80)

struct lzma_istream {
	struct istream_private istream;

	lzma_stream strm;
	uoff_t eof_offset, stream_size;
	size_t high_pos;
	struct stat last_parent_statbuf;

	unsigned int log_errors:1;
	unsigned int marked:1;
	unsigned int strm_closed:1;
};

static void i_stream_lzma_close(struct iostream_private *stream,
				 bool close_parent)
{
	struct lzma_istream *zstream = (struct lzma_istream *)stream;

	if (!zstream->strm_closed) {
		lzma_end(&zstream->strm);
		zstream->strm_closed = TRUE;
	}
	if (close_parent)
		i_stream_close(zstream->istream.parent);
}

static void lzma_read_error(struct lzma_istream *zstream, const char *error)
{
	io_stream_set_error(&zstream->istream.iostream,
			    "lzma.read(%s): %s at %"PRIuUOFF_T,
			    i_stream_get_name(&zstream->istream.istream), error,
			    zstream->istream.abs_start_offset +
			    zstream->istream.istream.v_offset);
	if (zstream->log_errors)
		i_error("%s", zstream->istream.iostream.error);
}

static ssize_t i_stream_lzma_read(struct istream_private *stream)
{
	struct lzma_istream *zstream = (struct lzma_istream *)stream;
	const unsigned char *data;
	uoff_t high_offset;
	size_t size, out_size;
	lzma_ret ret;

	high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
	if (zstream->eof_offset == high_offset) {
		i_assert(zstream->high_pos == 0 ||
			 zstream->high_pos == stream->pos);
		stream->istream.eof = TRUE;
		return -1;
	}

	if (stream->pos < zstream->high_pos) {
		/* we're here because we seeked back within the read buffer. */
		ret = zstream->high_pos - stream->pos;
		stream->pos = zstream->high_pos;
		zstream->high_pos = 0;

		if (zstream->eof_offset != (uoff_t)-1) {
			high_offset = stream->istream.v_offset +
				(stream->pos - stream->skip);
			i_assert(zstream->eof_offset == high_offset);
			stream->istream.eof = TRUE;
		}
		return ret;
	}
	zstream->high_pos = 0;

	if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
		/* try to keep at least CHUNK_SIZE available */
		if (!zstream->marked && stream->skip > 0) {
			/* don't try to keep anything cached if we don't
			   have a seek mark. */
			i_stream_compress(stream);
		}
		if (stream->max_buffer_size == 0 ||
		    stream->buffer_size < stream->max_buffer_size)
			i_stream_grow_buffer(stream, CHUNK_SIZE);

		if (stream->pos == stream->buffer_size) {
			if (stream->skip > 0) {
				/* lose our buffer cache */
				i_stream_compress(stream);
			}

			if (stream->pos == stream->buffer_size)
				return -2; /* buffer full */
		}
	}

	if (i_stream_read_data(stream->parent, &data, &size, 0) < 0) {
		if (stream->parent->stream_errno != 0) {
			stream->istream.stream_errno =
				stream->parent->stream_errno;
		} else {
			i_assert(stream->parent->eof);
			lzma_read_error(zstream, "unexpected EOF");
			stream->istream.stream_errno = EINVAL;
		}
		return -1;
	}
	if (size == 0) {
		/* no more input */
		i_assert(!stream->istream.blocking);
		return 0;
	}

	zstream->strm.next_in = data;
	zstream->strm.avail_in = size;

	out_size = stream->buffer_size - stream->pos;
	zstream->strm.next_out = stream->w_buffer + stream->pos;
	zstream->strm.avail_out = out_size;
	ret = lzma_code(&zstream->strm, LZMA_RUN);

	out_size -= zstream->strm.avail_out;
	stream->pos += out_size;

	i_stream_skip(stream->parent, size - zstream->strm.avail_in);

	switch (ret) {
	case LZMA_OK:
		break;
	case LZMA_DATA_ERROR:
	case LZMA_BUF_ERROR:
		lzma_read_error(zstream, "corrupted data");
		stream->istream.stream_errno = EINVAL;
		return -1;
	case LZMA_FORMAT_ERROR:
		lzma_read_error(zstream, "wrong magic in header (not xz file?)");
		stream->istream.stream_errno = EINVAL;
		return -1;
	case LZMA_OPTIONS_ERROR:
		lzma_read_error(zstream, "Unsupported xz options");
		stream->istream.stream_errno = EINVAL;
		return -1;
	case LZMA_MEM_ERROR:
		i_fatal_status(FATAL_OUTOFMEM, "lzma.read(%s): Out of memory",
			       i_stream_get_name(&stream->istream));
	case LZMA_STREAM_END:
		zstream->eof_offset = stream->istream.v_offset +
			(stream->pos - stream->skip);
		zstream->stream_size = zstream->eof_offset;
		if (out_size == 0) {
			stream->istream.eof = TRUE;
			return -1;
		}
		break;
	default:
		lzma_read_error(zstream, t_strdup_printf(
			"lzma_code() failed with %d", ret));
		stream->istream.stream_errno = EINVAL;
		return -1;
	}
	if (out_size == 0) {
		/* read more input */
		return i_stream_lzma_read(stream);
	}
	return out_size;
}

static void i_stream_lzma_init(struct lzma_istream *zstream)
{
	lzma_ret ret;

	ret = lzma_stream_decoder(&zstream->strm, LZMA_MEMORY_LIMIT,
				  LZMA_CONCATENATED);
	switch (ret) {
	case LZMA_OK:
		break;<commit_msg>lzma: Reading until EOF without seeing LZMA_STREAM_END isn't an error.<commit_after>/* Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file */

#include "lib.h"

#ifdef HAVE_LZMA

#include "istream-private.h"
#include "istream-zlib.h"
#include <lzma.h>

#define CHUNK_SIZE (1024*64)

#define LZMA_MEMORY_LIMIT (1024*1024*80)

struct lzma_istream {
	struct istream_private istream;

	lzma_stream strm;
	uoff_t eof_offset, stream_size;
	size_t high_pos;
	struct stat last_parent_statbuf;

	unsigned int log_errors:1;
	unsigned int marked:1;
	unsigned int strm_closed:1;
};

static void i_stream_lzma_close(struct iostream_private *stream,
				 bool close_parent)
{
	struct lzma_istream *zstream = (struct lzma_istream *)stream;

	if (!zstream->strm_closed) {
		lzma_end(&zstream->strm);
		zstream->strm_closed = TRUE;
	}
	if (close_parent)
		i_stream_close(zstream->istream.parent);
}

static void lzma_read_error(struct lzma_istream *zstream, const char *error)
{
	io_stream_set_error(&zstream->istream.iostream,
			    "lzma.read(%s): %s at %"PRIuUOFF_T,
			    i_stream_get_name(&zstream->istream.istream), error,
			    zstream->istream.abs_start_offset +
			    zstream->istream.istream.v_offset);
	if (zstream->log_errors)
		i_error("%s", zstream->istream.iostream.error);
}

static void lzma_stream_end(struct lzma_istream *zstream)
{
	zstream->eof_offset = zstream->istream.istream.v_offset +
		(zstream->istream.pos - zstream->istream.skip);
	zstream->stream_size = zstream->eof_offset;
}

static ssize_t i_stream_lzma_read(struct istream_private *stream)
{
	struct lzma_istream *zstream = (struct lzma_istream *)stream;
	const unsigned char *data;
	uoff_t high_offset;
	size_t size, out_size;
	lzma_ret ret;

	high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
	if (zstream->eof_offset == high_offset) {
		i_assert(zstream->high_pos == 0 ||
			 zstream->high_pos == stream->pos);
		stream->istream.eof = TRUE;
		return -1;
	}

	if (stream->pos < zstream->high_pos) {
		/* we're here because we seeked back within the read buffer. */
		ret = zstream->high_pos - stream->pos;
		stream->pos = zstream->high_pos;
		zstream->high_pos = 0;

		if (zstream->eof_offset != (uoff_t)-1) {
			high_offset = stream->istream.v_offset +
				(stream->pos - stream->skip);
			i_assert(zstream->eof_offset == high_offset);
			stream->istream.eof = TRUE;
		}
		return ret;
	}
	zstream->high_pos = 0;

	if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
		/* try to keep at least CHUNK_SIZE available */
		if (!zstream->marked && stream->skip > 0) {
			/* don't try to keep anything cached if we don't
			   have a seek mark. */
			i_stream_compress(stream);
		}
		if (stream->max_buffer_size == 0 ||
		    stream->buffer_size < stream->max_buffer_size)
			i_stream_grow_buffer(stream, CHUNK_SIZE);

		if (stream->pos == stream->buffer_size) {
			if (stream->skip > 0) {
				/* lose our buffer cache */
				i_stream_compress(stream);
			}

			if (stream->pos == stream->buffer_size)
				return -2; /* buffer full */
		}
	}

	if (i_stream_read_data(stream->parent, &data, &size, 0) < 0) {
		if (stream->parent->stream_errno != 0) {
			stream->istream.stream_errno =
				stream->parent->stream_errno;
		} else {
			i_assert(stream->parent->eof);
			lzma_stream_end(zstream);
			stream->istream.eof = TRUE;
		}
		return -1;
	}
	if (size == 0) {
		/* no more input */
		i_assert(!stream->istream.blocking);
		return 0;
	}

	zstream->strm.next_in = data;
	zstream->strm.avail_in = size;

	out_size = stream->buffer_size - stream->pos;
	zstream->strm.next_out = stream->w_buffer + stream->pos;
	zstream->strm.avail_out = out_size;
	ret = lzma_code(&zstream->strm, LZMA_RUN);

	out_size -= zstream->strm.avail_out;
	stream->pos += out_size;

	i_stream_skip(stream->parent, size - zstream->strm.avail_in);

	switch (ret) {
	case LZMA_OK:
		break;
	case LZMA_DATA_ERROR:
	case LZMA_BUF_ERROR:
		lzma_read_error(zstream, "corrupted data");
		stream->istream.stream_errno = EINVAL;
		return -1;
	case LZMA_FORMAT_ERROR:
		lzma_read_error(zstream, "wrong magic in header (not xz file?)");
		stream->istream.stream_errno = EINVAL;
		return -1;
	case LZMA_OPTIONS_ERROR:
		lzma_read_error(zstream, "Unsupported xz options");
		stream->istream.stream_errno = EINVAL;
		return -1;
	case LZMA_MEM_ERROR:
		i_fatal_status(FATAL_OUTOFMEM, "lzma.read(%s): Out of memory",
			       i_stream_get_name(&stream->istream));
	case LZMA_STREAM_END:
		lzma_stream_end(zstream);
		if (out_size == 0) {
			stream->istream.eof = TRUE;
			return -1;
		}
		break;
	default:
		lzma_read_error(zstream, t_strdup_printf(
			"lzma_code() failed with %d", ret));
		stream->istream.stream_errno = EINVAL;
		return -1;
	}
	if (out_size == 0) {
		/* read more input */
		return i_stream_lzma_read(stream);
	}
	return out_size;
}

static void i_stream_lzma_init(struct lzma_istream *zstream)
{
	lzma_ret ret;

	ret = lzma_stream_decoder(&zstream->strm, LZMA_MEMORY_LIMIT,
				  LZMA_CONCATENATED);
	switch (ret) {
	case LZMA_OK:
		break;
{'fraction_non_alphanumeric': 0.13515695972544664, 'fraction_numerical': 0.009891995558695871, 'mean_word_length': 4.133678756476684, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 171, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6940902', 'n_tokens_mistral': 4242, 'n_tokens_neox': 3887, 'n_words': 756}