<commit_before>/* Copyright (C) 2002 <NAME> */

#include "lib.h"
#include "base64.h"
#include "buffer.h"
#include "istream.h"
#include "strescape.h"
#include "charset-utf8.h"
#include "quoted-printable.h"
#include "message-parser.h"
#include "message-content-parser.h"
#include "message-header-search.h"
#include "message-body-search.h"

#define DECODE_BLOCK_SIZE 8192

struct body_search_context {
	pool_t pool;

	const char *key;
	size_t key_len;

	const char *charset;
	unsigned int unknown_charset:1;
	unsigned int search_header:1;
};

struct part_search_context {
	struct body_search_context *body_ctx;

	struct charset_translation *translation;

	buffer_t *decode_buf;
	buffer_t *match_buf;

	char *content_type;
	char *content_charset;

	unsigned int content_qp:1;
	unsigned int content_base64:1;
	unsigned int content_unknown:1;
	unsigned int content_type_text:1; /* text/any or message/any */
	unsigned int ignore_header:1;
};

static void parse_content_type(const unsigned char *value, size_t value_len,
			       void *context)
{
	struct part_search_context *ctx = context;

	if (ctx->content_type == NULL) {
		ctx->content_type = i_strndup(value, value_len);
		ctx->content_type_text =
			strncasecmp(ctx->content_type, "text/", 5) == 0 ||
			strncasecmp(ctx->content_type, "message/", 8) == 0;
	}
}

static void
parse_content_type_param(const unsigned char *name, size_t name_len,
			 const unsigned char *value, size_t value_len,
			 int value_quoted, void *context)
{
	struct part_search_context *ctx = context;

	if (name_len == 7 && memcasecmp(name, "charset", 7) == 0 &&
	    ctx->content_charset == NULL) {
		ctx->content_charset = i_strndup(value, value_len);
		if (value_quoted) str_unescape(ctx->content_charset);
	}
}

static void parse_content_encoding(const unsigned char *value, size_t value_len,
				   void *context)
{
	struct part_search_context *ctx = context;

	switch (value_len) {
	case 4:
		if (memcasecmp(value, "7bit", 4) != 0 &&
		    memcasecmp(value, "8bit", 4) != 0)
			ctx->content_unknown = TRUE;
		break;
	case 6:
		if (memcasecmp(value, "base64", 6) == 0)
			ctx->content_base64 = TRUE;
		else if (memcasecmp(value, "binary", 6) != 0)
			ctx->content_unknown = TRUE;
		break;
	case 16:
		if (memcasecmp(value, "quoted-printable", 16) == 0)
			ctx->content_qp = TRUE;
		else
			ctx->content_unknown = TRUE;
		break;
	default:
		ctx->content_unknown = TRUE;
		break;
	}
}

static int message_search_header(struct part_search_context *ctx,
				 struct istream *input)
{
	struct header_search_context *hdr_search_ctx;
	struct message_header_parser_ctx *hdr_ctx;
	struct message_header_line *hdr;
	int ret, found = FALSE;

	hdr_search_ctx = message_header_search_init(pool_datastack_create(),
						    ctx->body_ctx->key,
						    ctx->body_ctx->charset,
						    NULL);

	/* we default to text content-type */
	ctx->content_type_text = TRUE;

	hdr_ctx = message_parse_header_init(input, NULL, TRUE);
	while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) {
		if (hdr->eoh)
			continue;

		if (!ctx->ignore_header) {
			if (message_header_search(hdr->value, hdr->value_len,
						  hdr_search_ctx)) {
				found = TRUE;
				break;
			}
		}

		if (hdr->name_len == 12 &&
		    strcasecmp(hdr->name, "Content-Type") == 0) {
			if (hdr->continues) {
				hdr->use_full_value = TRUE;
				continue;
			}
			message_content_parse_header(hdr->full_value,
						     hdr->full_value_len,
						     parse_content_type,
						     parse_content_type_param,
						     ctx);
		} else if (hdr->name_len == 25 &&
			   strcasecmp(hdr->name,
				      "Content-Transfer-Encoding") == 0) {
			if (hdr->continues) {
				hdr->use_full_value = TRUE;
				continue;
			}
			message_content_parse_header(hdr->full_value,
						     hdr->full_value_len,
						     parse_content_encoding,
						     NULL, ctx);
		}
	}
	i_assert(ret != 0);
	message_parse_header_deinit(hdr_ctx);

	return found;
}

static int message_search_decoded_block(struct part_search_context *ctx,
					buffer_t *block)
{
	const unsigned char *p, *end, *key;
	size_t key_len, block_size, *matches, match_count, value;
	ssize_t i;

	key = (const unsigned char *) ctx->body_ctx->key;
	key_len = ctx->body_ctx->key_len;

	matches = buffer_get_modifyable_data(ctx->match_buf, &match_count);
	match_count /= sizeof(size_t);

	p = buffer_get_data(block, &block_size);
	end = p + block_size;
	for (; p != end; p++) {
		for (i = match_count-1; i >= 0; i--) {
			if (key[matches[i]] == *p) {
				if (++matches[i] == key_len) {
					/* full match */
					p++;
					return TRUE;
				}
			} else {
				/* non-match */
				buffer_delete(ctx->match_buf,
					      i * sizeof(size_t),
					      sizeof(size_t));
				match_count--;
			}
		}

		if (*p == key[0]) {
			if (key_len == 1) {
				/* only one character in search key */
				p++;
				return TRUE;
			}

			value = 1;
			buffer_append(ctx->match_buf, &value, sizeof(value));
			match_count++;
		}
	}

	return FALSE;
}

/* returns 1 = found, 0 = not found, -1 = error in input data */
static int message_search_body_block(struct part_search_context *ctx,
				     buffer_t *block)
{
	const unsigned char *inbuf;
	buffer_t *outbuf;
        enum charset_result result;
	size_t block_pos, inbuf_size, inbuf_left;

	outbuf = buffer_create_static_hard(pool_datastack_create(),
					   DECODE_BLOCK_SIZE);
	for (block_pos = 0; block_pos < buffer_get_used_size(block); ) {
		if (buffer_get_used_size(ctx->decode_buf) == 0) {
			/* we can use the buffer directly without copying */
			inbuf = buffer_get_data(block, &inbuf_size);
			inbuf += block_pos; inbuf_size -= block_pos;
			block_pos += buffer_get_used_size(block);
		} else {
			/* some characters already in buffer, ie. last
			   conversion contained partial data */
			buffer_append_buf(ctx->decode_buf, block,
					  block_pos, block->used);
                        block_pos += block->used;

			inbuf = buffer_get_data(ctx->decode_buf, &inbuf_size);
		}

		buffer_set_used_size(outbuf, 0);
		inbuf_left = inbuf_size;
		result = charset_to_ucase_utf8(ctx->translation,
					       inbuf, &inbuf_size, outbuf);
		inbuf_left -= inbuf_size;

		switch (result) {
		case CHARSET_RET_OUTPUT_FULL:
			/* we should have copied the incomplete sequence.. */
			i_assert(inbuf_left <= block_pos);
			/* fall through */
		case CHARSET_RET_OK:
			buffer_set_used_size(ctx->decode_buf, 0);
			block_pos -= inbuf_left;
			break;
		case CHARSET_RET_INCOMPLETE_INPUT:
			/* save the partial sequence to buffer */
			buffer_write(ctx->decode_buf, 0,
				     inbuf + inbuf_size, inbuf_left);
			buffer_set_used_size(ctx->decode_buf, inbuf_left);
			break;

		case CHARSET_RET_INVALID_INPUT:
			return -1;
		}

		if (message_search_decoded_block(ctx, outbuf))
			return 1;
	}

	return 0;
}

static int message_search_body(struct part_search_context *ctx,
			       struct istream *input,
			       const struct message_part *part)
{
	const unsigned char *data;
	buffer_t *decodebuf;
	pool_t pool;
	size_t data_size, pos;
	ssize_t ret;
	int found;

	if (ctx->content_unknown) {
		/* unknown content-encoding-type, ignore */
		return FALSE;
	}

	if (!ctx->content_type_text) {
		/* non-text content, ignore - FIXME: should be configurable? */
		return FALSE;
	}

	ctx->translation = ctx->content_charset == NULL ? NULL :
		charset_to_utf8_begin(ctx->content_charset, NULL);
	if (ctx->translation == NULL)
		ctx->translation = charset_to_utf8_begin("ascii", NULL);

	ctx->decode_buf =
		buffer_create_static_hard(pool_datastack_create(), 256);
	ctx->match_buf = buffer_create_static_hard(pool_datastack_create(),
						   sizeof(size_t) *
						   ctx->body_ctx->key_len);

	input = i_stream_create_limit(default_pool, input,
				      part->physical_pos +
				      part->header_size.physical_size,
				      part->body_size.physical_size);
	i_stream_seek(input, 0);

	found = FALSE; pos = 0;
	while (i_stream_read_data(input, &data, &data_size, pos) > 0) {
		/* limit the size of t_malloc()s */
		if (data_size > DECODE_BLOCK_SIZE)
			data_size = DECODE_BLOCK_SIZE;
		pos = data_size;

		t_push();
		pool = pool_datastack_create();
		if (ctx->content_qp) {<commit_msg>Use dynamic buffer. Probably fixes some "buffer full" crashes.<commit_after>/* Copyright (C) 2002 <NAME> */

#include "lib.h"
#include "base64.h"
#include "buffer.h"
#include "istream.h"
#include "strescape.h"
#include "charset-utf8.h"
#include "quoted-printable.h"
#include "message-parser.h"
#include "message-content-parser.h"
#include "message-header-search.h"
#include "message-body-search.h"

#define DECODE_BLOCK_SIZE 8192

struct body_search_context {
	pool_t pool;

	const char *key;
	size_t key_len;

	const char *charset;
	unsigned int unknown_charset:1;
	unsigned int search_header:1;
};

struct part_search_context {
	struct body_search_context *body_ctx;

	struct charset_translation *translation;

	buffer_t *decode_buf;
	buffer_t *match_buf;

	char *content_type;
	char *content_charset;

	unsigned int content_qp:1;
	unsigned int content_base64:1;
	unsigned int content_unknown:1;
	unsigned int content_type_text:1; /* text/any or message/any */
	unsigned int ignore_header:1;
};

static void parse_content_type(const unsigned char *value, size_t value_len,
			       void *context)
{
	struct part_search_context *ctx = context;

	if (ctx->content_type == NULL) {
		ctx->content_type = i_strndup(value, value_len);
		ctx->content_type_text =
			strncasecmp(ctx->content_type, "text/", 5) == 0 ||
			strncasecmp(ctx->content_type, "message/", 8) == 0;
	}
}

static void
parse_content_type_param(const unsigned char *name, size_t name_len,
			 const unsigned char *value, size_t value_len,
			 int value_quoted, void *context)
{
	struct part_search_context *ctx = context;

	if (name_len == 7 && memcasecmp(name, "charset", 7) == 0 &&
	    ctx->content_charset == NULL) {
		ctx->content_charset = i_strndup(value, value_len);
		if (value_quoted) str_unescape(ctx->content_charset);
	}
}

static void parse_content_encoding(const unsigned char *value, size_t value_len,
				   void *context)
{
	struct part_search_context *ctx = context;

	switch (value_len) {
	case 4:
		if (memcasecmp(value, "7bit", 4) != 0 &&
		    memcasecmp(value, "8bit", 4) != 0)
			ctx->content_unknown = TRUE;
		break;
	case 6:
		if (memcasecmp(value, "base64", 6) == 0)
			ctx->content_base64 = TRUE;
		else if (memcasecmp(value, "binary", 6) != 0)
			ctx->content_unknown = TRUE;
		break;
	case 16:
		if (memcasecmp(value, "quoted-printable", 16) == 0)
			ctx->content_qp = TRUE;
		else
			ctx->content_unknown = TRUE;
		break;
	default:
		ctx->content_unknown = TRUE;
		break;
	}
}

static int message_search_header(struct part_search_context *ctx,
				 struct istream *input)
{
	struct header_search_context *hdr_search_ctx;
	struct message_header_parser_ctx *hdr_ctx;
	struct message_header_line *hdr;
	int ret, found = FALSE;

	hdr_search_ctx = message_header_search_init(pool_datastack_create(),
						    ctx->body_ctx->key,
						    ctx->body_ctx->charset,
						    NULL);

	/* we default to text content-type */
	ctx->content_type_text = TRUE;

	hdr_ctx = message_parse_header_init(input, NULL, TRUE);
	while ((ret = message_parse_header_next(hdr_ctx, &hdr)) > 0) {
		if (hdr->eoh)
			continue;

		if (!ctx->ignore_header) {
			if (message_header_search(hdr->value, hdr->value_len,
						  hdr_search_ctx)) {
				found = TRUE;
				break;
			}
		}

		if (hdr->name_len == 12 &&
		    strcasecmp(hdr->name, "Content-Type") == 0) {
			if (hdr->continues) {
				hdr->use_full_value = TRUE;
				continue;
			}
			message_content_parse_header(hdr->full_value,
						     hdr->full_value_len,
						     parse_content_type,
						     parse_content_type_param,
						     ctx);
		} else if (hdr->name_len == 25 &&
			   strcasecmp(hdr->name,
				      "Content-Transfer-Encoding") == 0) {
			if (hdr->continues) {
				hdr->use_full_value = TRUE;
				continue;
			}
			message_content_parse_header(hdr->full_value,
						     hdr->full_value_len,
						     parse_content_encoding,
						     NULL, ctx);
		}
	}
	i_assert(ret != 0);
	message_parse_header_deinit(hdr_ctx);

	return found;
}

static int message_search_decoded_block(struct part_search_context *ctx,
					buffer_t *block)
{
	const unsigned char *p, *end, *key;
	size_t key_len, block_size, *matches, match_count, value;
	ssize_t i;

	key = (const unsigned char *) ctx->body_ctx->key;
	key_len = ctx->body_ctx->key_len;

	matches = buffer_get_modifyable_data(ctx->match_buf, &match_count);
	match_count /= sizeof(size_t);

	p = buffer_get_data(block, &block_size);
	end = p + block_size;
	for (; p != end; p++) {
		for (i = match_count-1; i >= 0; i--) {
			if (key[matches[i]] == *p) {
				if (++matches[i] == key_len) {
					/* full match */
					p++;
					return TRUE;
				}
			} else {
				/* non-match */
				buffer_delete(ctx->match_buf,
					      i * sizeof(size_t),
					      sizeof(size_t));
				match_count--;
			}
		}

		if (*p == key[0]) {
			if (key_len == 1) {
				/* only one character in search key */
				p++;
				return TRUE;
			}

			value = 1;
			buffer_append(ctx->match_buf, &value, sizeof(value));
			match_count++;
		}
	}

	return FALSE;
}

/* returns 1 = found, 0 = not found, -1 = error in input data */
static int message_search_body_block(struct part_search_context *ctx,
				     buffer_t *block)
{
	const unsigned char *inbuf;
	buffer_t *outbuf;
        enum charset_result result;
	size_t block_pos, inbuf_size, inbuf_left;

	outbuf = buffer_create_static_hard(pool_datastack_create(),
					   DECODE_BLOCK_SIZE);
	for (block_pos = 0; block_pos < buffer_get_used_size(block); ) {
		if (buffer_get_used_size(ctx->decode_buf) == 0) {
			/* we can use the buffer directly without copying */
			inbuf = buffer_get_data(block, &inbuf_size);
			inbuf += block_pos; inbuf_size -= block_pos;
			block_pos += buffer_get_used_size(block);
		} else {
			/* some characters already in buffer, ie. last
			   conversion contained partial data */
			buffer_append_buf(ctx->decode_buf, block,
					  block_pos, block->used);
                        block_pos += block->used;

			inbuf = buffer_get_data(ctx->decode_buf, &inbuf_size);
		}

		buffer_set_used_size(outbuf, 0);
		inbuf_left = inbuf_size;
		result = charset_to_ucase_utf8(ctx->translation,
					       inbuf, &inbuf_size, outbuf);
		inbuf_left -= inbuf_size;

		switch (result) {
		case CHARSET_RET_OUTPUT_FULL:
			/* we should have copied the incomplete sequence.. */
			i_assert(inbuf_left <= block_pos);
			/* fall through */
		case CHARSET_RET_OK:
			buffer_set_used_size(ctx->decode_buf, 0);
			block_pos -= inbuf_left;
			break;
		case CHARSET_RET_INCOMPLETE_INPUT:
			/* save the partial sequence to buffer */
			buffer_write(ctx->decode_buf, 0,
				     inbuf + inbuf_size, inbuf_left);
			buffer_set_used_size(ctx->decode_buf, inbuf_left);
			break;

		case CHARSET_RET_INVALID_INPUT:
			return -1;
		}

		if (message_search_decoded_block(ctx, outbuf))
			return 1;
	}

	return 0;
}

static int message_search_body(struct part_search_context *ctx,
			       struct istream *input,
			       const struct message_part *part)
{
	const unsigned char *data;
	buffer_t *decodebuf;
	pool_t pool;
	size_t data_size, pos;
	ssize_t ret;
	int found;

	if (ctx->content_unknown) {
		/* unknown content-encoding-type, ignore */
		return FALSE;
	}

	if (!ctx->content_type_text) {
		/* non-text content, ignore - FIXME: should be configurable? */
		return FALSE;
	}

	ctx->translation = ctx->content_charset == NULL ? NULL :
		charset_to_utf8_begin(ctx->content_charset, NULL);
	if (ctx->translation == NULL)
		ctx->translation = charset_to_utf8_begin("ascii", NULL);

	ctx->decode_buf =
		buffer_create_dynamic(pool_datastack_create(), 256);
	ctx->match_buf = buffer_create_static_hard(pool_datastack_create(),
						   sizeof(size_t) *
						   ctx->body_ctx->key_len);

	input = i_stream_create_limit(default_pool, input,
				      part->physical_pos +
				      part->header_size.physical_size,
				      part->body_size.physical_size);
	i_stream_seek(input, 0);

	found = FALSE; pos = 0;
	while (i_stream_read_data(input, &data, &data_size, pos) > 0) {
		/* limit the size of t_malloc()s */
		if (data_size > DECODE_BLOCK_SIZE)
			data_size = DECODE_BLOCK_SIZE;
		pos = data_size;

		t_push();
		pool = pool_datastack_create();
		if (ctx->content_qp) {
{'fraction_non_alphanumeric': 0.12392089634482337, 'fraction_numerical': 0.009918569766729933, 'mean_word_length': 3.625885018408383, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5696970', 'n_tokens_mistral': 6730, 'n_tokens_neox': 5777, 'n_words': 1276}