<commit_before>/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */

#include "lib.h"

#ifdef HAVE_LZ4

#include "ostream-private.h"
#include "ostream-zlib.h"
#include "iostream-lz4.h"
#include <lz4.h>

#define CHUNK_SIZE OSTREAM_LZ4_CHUNK_SIZE

struct lz4_ostream {
	struct ostream_private ostream;

	unsigned char compressbuf[CHUNK_SIZE];
	unsigned int compressbuf_offset;

	/* chunk size, followed by compressed data */
	unsigned char outbuf[IOSTREAM_LZ4_CHUNK_PREFIX_LEN + LZ4_COMPRESSBOUND(CHUNK_SIZE)];
	unsigned int outbuf_offset, outbuf_used;
};

static void o_stream_lz4_close(struct iostream_private *stream,
			       bool close_parent)
{
	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;

	(void)o_stream_flush(&zstream->ostream.ostream);
	if (close_parent)
		o_stream_close(zstream->ostream.parent);
}

static int o_stream_lz4_send_outbuf(struct lz4_ostream *zstream)
{
	ssize_t ret;
	size_t size;

	if (zstream->outbuf_used == 0)
		return 1;

	size = zstream->outbuf_used - zstream->outbuf_offset;
	i_assert(size > 0);
	ret = o_stream_send(zstream->ostream.parent,
			    zstream->outbuf + zstream->outbuf_offset, size);
	if (ret < 0) {
		o_stream_copy_error_from_parent(&zstream->ostream);
		return -1;
	}
	if ((size_t)ret != size) {
		zstream->outbuf_offset += ret;
		return 0;
	}
	zstream->outbuf_offset = 0;
	zstream->outbuf_used = 0;
	return 1;
}

static int o_stream_lz4_compress(struct lz4_ostream *zstream)
{
	uint32_t chunk_size;
	int ret;

	if (zstream->compressbuf_offset == 0)
		return 1;
	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0)
		return ret;

	i_assert(zstream->outbuf_offset == 0);
	i_assert(zstream->outbuf_used == 0);

	zstream->outbuf_used = IOSTREAM_LZ4_CHUNK_PREFIX_LEN +
		LZ4_compress((void *)zstream->compressbuf,
			     (void *)(zstream->outbuf + IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
			     zstream->compressbuf_offset);
	i_assert(zstream->outbuf_used > IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
	chunk_size = zstream->outbuf_used - IOSTREAM_LZ4_CHUNK_PREFIX_LEN;
	zstream->outbuf[0] = (chunk_size & 0xff000000) >> 24;
	zstream->outbuf[1] = (chunk_size & 0x00ff0000) >> 16;
	zstream->outbuf[2] = (chunk_size & 0x0000ff00) >> 8;
	zstream->outbuf[3] = (chunk_size & 0x000000ff);
	zstream->compressbuf_offset = 0;
	return 1;
}<commit_msg>lib-compression: Add assert for LZ4_compress return value. Make certain we detect if compressed data overflows the allocated space.<commit_after>/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */

#include "lib.h"

#ifdef HAVE_LZ4

#include "ostream-private.h"
#include "ostream-zlib.h"
#include "iostream-lz4.h"
#include <lz4.h>

#define CHUNK_SIZE OSTREAM_LZ4_CHUNK_SIZE

struct lz4_ostream {
	struct ostream_private ostream;

	unsigned char compressbuf[CHUNK_SIZE];
	unsigned int compressbuf_offset;

	/* chunk size, followed by compressed data */
	unsigned char outbuf[IOSTREAM_LZ4_CHUNK_PREFIX_LEN +
	                     LZ4_COMPRESSBOUND(CHUNK_SIZE)];
	unsigned int outbuf_offset, outbuf_used;
};

static void o_stream_lz4_close(struct iostream_private *stream,
			       bool close_parent)
{
	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;

	(void)o_stream_flush(&zstream->ostream.ostream);
	if (close_parent)
		o_stream_close(zstream->ostream.parent);
}

static int o_stream_lz4_send_outbuf(struct lz4_ostream *zstream)
{
	ssize_t ret;
	size_t size;

	if (zstream->outbuf_used == 0)
		return 1;

	size = zstream->outbuf_used - zstream->outbuf_offset;
	i_assert(size > 0);
	ret = o_stream_send(zstream->ostream.parent,
			    zstream->outbuf + zstream->outbuf_offset, size);
	if (ret < 0) {
		o_stream_copy_error_from_parent(&zstream->ostream);
		return -1;
	}
	if ((size_t)ret != size) {
		zstream->outbuf_offset += ret;
		return 0;
	}
	zstream->outbuf_offset = 0;
	zstream->outbuf_used = 0;
	return 1;
}

static int o_stream_lz4_compress(struct lz4_ostream *zstream)
{
	uint32_t chunk_size;
	int ret;

	if (zstream->compressbuf_offset == 0)
		return 1;
	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0)
		return ret;

	i_assert(zstream->outbuf_offset == 0);
	i_assert(zstream->outbuf_used == 0);

	ret = LZ4_compress((void *)zstream->compressbuf,
			   (void *)(zstream->outbuf +
			            IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
			   zstream->compressbuf_offset);
	i_assert(ret > 0 && (unsigned int)ret <= sizeof(zstream->outbuf) -
	         IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
	zstream->outbuf_used = IOSTREAM_LZ4_CHUNK_PREFIX_LEN + ret;
	chunk_size = zstream->outbuf_used - IOSTREAM_LZ4_CHUNK_PREFIX_LEN;
	zstream->outbuf[0] = (chunk_size & 0xff000000) >> 24;
	zstream->outbuf[1] = (chunk_size & 0x00ff0000) >> 16;
	zstream->outbuf[2] = (chunk_size & 0x0000ff00) >> 8;
	zstream->outbuf[3] = (chunk_size & 0x000000ff);
	zstream->compressbuf_offset = 0;
	return 1;
}
{'fraction_non_alphanumeric': 0.11970954356846474, 'fraction_numerical': 0.034854771784232366, 'mean_word_length': 4.981389578163772, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2343605', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1879, 'n_words': 357}