<commit_before>/* Copyright (c) 2002-2012 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "buffer.h"
#include "istream.h"
#include "str.h"
#include "message-size.h"
#include "message-header-parser.h"

struct message_header_parser_ctx {
	struct message_header_line line;

	struct istream *input;
	struct message_size *hdr_size;

	string_t *name;
	buffer_t *value_buf;
	size_t skip;

	enum message_header_parser_flags flags;
	unsigned int skip_line:1;
	unsigned int has_nuls:1;
};

struct message_header_parser_ctx *
message_parse_header_init(struct istream *input, struct message_size *hdr_size,
			  enum message_header_parser_flags flags)
{
	struct message_header_parser_ctx *ctx;

	ctx = i_new(struct message_header_parser_ctx, 1);
	ctx->input = input;
	ctx->hdr_size = hdr_size;
	ctx->name = str_new(default_pool, 128);
	ctx->flags = flags;
	ctx->value_buf = buffer_create_dynamic(default_pool, 4096);

	if (hdr_size != NULL)
		memset(hdr_size, 0, sizeof(*hdr_size));
	return ctx;
}

void message_parse_header_deinit(struct message_header_parser_ctx **_ctx)
{
	struct message_header_parser_ctx *ctx = *_ctx;

	i_stream_skip(ctx->input, ctx->skip);
	buffer_free(&ctx->value_buf);
	str_free(&ctx->name);
	i_free(ctx);

	*_ctx = NULL;
}

int message_parse_header_next(struct message_header_parser_ctx *ctx,
			      struct message_header_line **hdr_r)
{
        struct message_header_line *line = &ctx->line;
	const unsigned char *msg;
	size_t i, size, startpos, colon_pos, parse_size;
	int ret;
	bool continued, continues, last_no_newline, last_crlf;
	bool no_newline, crlf_newline;

	*hdr_r = NULL;
	if (line->eoh)
		return -1;

	if (ctx->skip > 0) {
		i_stream_skip(ctx->input, ctx->skip);
		ctx->skip = 0;
	}

	if (line->continues)
		colon_pos = 0;
	else {
		/* new header line */
		line->name_offset = ctx->input->v_offset;
		colon_pos = UINT_MAX;
		buffer_set_used_size(ctx->value_buf, 0);
	}

	no_newline = FALSE;
	crlf_newline = FALSE;
	continued = line->continues;
	continues = FALSE;

	for (startpos = 0;;) {
		ret = i_stream_read_data(ctx->input, &msg, &size, startpos+1);
		if (ret >= 0) {
			/* we want to know one byte in advance to find out
			   if it's multiline header */
			parse_size = size == 0 ? 0 : size-1;
		} else {
			parse_size = size;
		}

		if (ret <= 0 && startpos == parse_size) {
			if (ret == -1) {
				if (startpos > 0) {
					/* header ended unexpectedly. */
					no_newline = TRUE;
					ctx->skip = startpos;
					break;
				}
				/* error / EOF with no bytes */
				return -1;
			}

			if (size > 0 && !ctx->skip_line &&
			    (msg[0] == '\n' ||
			     (msg[0] == '\r' && size > 1 && msg[1] == '\n'))) {
				/* end of headers - this mostly happens just
				   with mbox where headers are read separately
				   from body */
				size = 0;
				if (ctx->hdr_size != NULL)
					ctx->hdr_size->lines++;
				if (msg[0] == '\r') {
					ctx->skip = 2;
					crlf_newline = TRUE;
				} else {
					ctx->skip = 1;
					if (ctx->hdr_size != NULL)
						ctx->hdr_size->virtual_size++;
				}
				break;
			}
			if (ret == 0 && !ctx->input->eof) {
				/* stream is nonblocking - need more data */
				return 0;
			}
			i_assert(size > 0);

			/* a) line is larger than input buffer
			   b) header ended unexpectedly */
			if (colon_pos == UINT_MAX && ret == -2 && !continued) {
				/* header name is huge. just skip it. */
				i_assert(size > 1);
				if (msg[size-1] == '\r')
					size--;

				if (ctx->hdr_size != NULL) {
					ctx->hdr_size->physical_size += size;
					ctx->hdr_size->virtual_size += size;
				}
				i_stream_skip(ctx->input, size);
				ctx->skip_line = TRUE;
				startpos = 0;
				continue;
			}

			if (ret == -2) {
				/* go back to last LWSP if found. */
				size_t min_pos = !continued ? colon_pos : 0;
				for (i = size-1; i > min_pos; i--) {
					if (IS_LWSP(msg[i])) {
						size = i;
						break;
					}
				}
				if (i == min_pos && (msg[size-1] == '\r' ||<commit_msg>message header parser: Don't skip header if header name is longer than input buffer. We'll instead now just return the name in smaller blocks with no_newline=TRUE. This allows the caller to reconstruct the exact same message.<commit_after>/* Copyright (c) 2002-2012 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "buffer.h"
#include "istream.h"
#include "str.h"
#include "message-size.h"
#include "message-header-parser.h"

struct message_header_parser_ctx {
	struct message_header_line line;

	struct istream *input;
	struct message_size *hdr_size;

	string_t *name;
	buffer_t *value_buf;
	size_t skip;

	enum message_header_parser_flags flags;
	unsigned int skip_line:1;
	unsigned int has_nuls:1;
};

struct message_header_parser_ctx *
message_parse_header_init(struct istream *input, struct message_size *hdr_size,
			  enum message_header_parser_flags flags)
{
	struct message_header_parser_ctx *ctx;

	ctx = i_new(struct message_header_parser_ctx, 1);
	ctx->input = input;
	ctx->hdr_size = hdr_size;
	ctx->name = str_new(default_pool, 128);
	ctx->flags = flags;
	ctx->value_buf = buffer_create_dynamic(default_pool, 4096);

	if (hdr_size != NULL)
		memset(hdr_size, 0, sizeof(*hdr_size));
	return ctx;
}

void message_parse_header_deinit(struct message_header_parser_ctx **_ctx)
{
	struct message_header_parser_ctx *ctx = *_ctx;

	i_stream_skip(ctx->input, ctx->skip);
	buffer_free(&ctx->value_buf);
	str_free(&ctx->name);
	i_free(ctx);

	*_ctx = NULL;
}

int message_parse_header_next(struct message_header_parser_ctx *ctx,
			      struct message_header_line **hdr_r)
{
        struct message_header_line *line = &ctx->line;
	const unsigned char *msg;
	size_t i, size, startpos, colon_pos, parse_size;
	int ret;
	bool continued, continues, last_no_newline, last_crlf;
	bool no_newline, crlf_newline;

	*hdr_r = NULL;
	if (line->eoh)
		return -1;

	if (ctx->skip > 0) {
		i_stream_skip(ctx->input, ctx->skip);
		ctx->skip = 0;
	}

	if (line->continues)
		colon_pos = 0;
	else {
		/* new header line */
		line->name_offset = ctx->input->v_offset;
		colon_pos = UINT_MAX;
		buffer_set_used_size(ctx->value_buf, 0);
	}

	no_newline = FALSE;
	crlf_newline = FALSE;
	continued = line->continues;
	continues = FALSE;

	for (startpos = 0;;) {
		ret = i_stream_read_data(ctx->input, &msg, &size, startpos+1);
		if (ret >= 0) {
			/* we want to know one byte in advance to find out
			   if it's multiline header */
			parse_size = size == 0 ? 0 : size-1;
		} else {
			parse_size = size;
		}

		if (ret <= 0 && startpos == parse_size) {
			if (ret == -1) {
				if (startpos > 0) {
					/* header ended unexpectedly. */
					no_newline = TRUE;
					ctx->skip = startpos;
					break;
				}
				/* error / EOF with no bytes */
				return -1;
			}

			if (size > 0 && !ctx->skip_line &&
			    (msg[0] == '\n' ||
			     (msg[0] == '\r' && size > 1 && msg[1] == '\n'))) {
				/* end of headers - this mostly happens just
				   with mbox where headers are read separately
				   from body */
				size = 0;
				if (ctx->hdr_size != NULL)
					ctx->hdr_size->lines++;
				if (msg[0] == '\r') {
					ctx->skip = 2;
					crlf_newline = TRUE;
				} else {
					ctx->skip = 1;
					if (ctx->hdr_size != NULL)
						ctx->hdr_size->virtual_size++;
				}
				break;
			}
			if (ret == 0 && !ctx->input->eof) {
				/* stream is nonblocking - need more data */
				return 0;
			}
			i_assert(size > 0);

			/* a) line is larger than input buffer
			   b) header ended unexpectedly */
			if (ret == -2) {
				/* go back to last LWSP if found. */
				size_t min_pos = !continued ? colon_pos : 0;
				for (i = size-1; i > min_pos; i--) {
					if (IS_LWSP(msg[i])) {
						size = i;
						break;
					}
				}
				if (i == min_pos && (msg[size-1] == '\r' ||
{'fraction_non_alphanumeric': 0.1408943616331821, 'fraction_numerical': 0.013480233311730396, 'mean_word_length': 3.065331928345627, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1541877', 'n_tokens_mistral': 3336, 'n_tokens_neox': 2805, 'n_words': 734}