<commit_before>/* Copyright (C) 2002 <NAME> */

#include "lib.h"
#include "istream.h"
#include "hex-binary.h"
#include "message-parser.h"
#include "message-part-serialize.h"
#include "mbox-index.h"
#include "mbox-lock.h"
#include "mail-index-util.h"

#include <unistd.h>
#include <fcntl.h>

static void skip_line(IStream *input)
{
	const unsigned char *msg;
	size_t i, size;

	while (i_stream_read_data(input, &msg, &size, 0) > 0) {
		for (i = 0; i < size; i++) {
			if (msg[i] == '\n') {
				i_stream_skip(input, i+1);
				return;
			}
		}

		i_stream_skip(input, i);
	}
}

static int verify_header_md5sum(MailIndex *index, MailIndexRecord *rec,
				unsigned char current_digest[16])
{
	const unsigned char *old_digest;
	size_t size;

	/* MD5 sums must match */
	old_digest = index->lookup_field_raw(index, rec, DATA_FIELD_MD5, &size);
	return old_digest != NULL && size >= 16 &&
                memcmp(old_digest, current_digest, 16) == 0;
}

static int mail_update_header_size(MailIndex *index, MailIndexRecord *rec,
				   MailIndexUpdate *update,
				   MessageSize *hdr_size)
{
	const void *part_data;
	void *part_data_copy;
	uoff_t virtual_size;
	size_t size;

	/* update FIELD_HDR_HEADER_SIZE */
	index->update_field_raw(update, DATA_HDR_HEADER_SIZE,
				&hdr_size->physical_size,
				sizeof(hdr_size->physical_size));

	/* reset FIELD_HDR_VIRTUAL_SIZE - we don't know it anymore */
        virtual_size = (uoff_t)-1;
	index->update_field_raw(update, DATA_HDR_VIRTUAL_SIZE,
				&virtual_size, sizeof(virtual_size));

	/* update DATA_FIELD_MESSAGEPART */
	if ((rec->data_fields & DATA_FIELD_MESSAGEPART) == 0)
		return TRUE;

	part_data = index->lookup_field_raw(index, rec, DATA_FIELD_MESSAGEPART,
					    &size);
	if (part_data == NULL) {
		/* well, this wasn't expected but don't bother failing */
		return TRUE;
	}

	t_push();

	/* copy & update the part data */
	part_data_copy = t_malloc(size);
	memcpy(part_data_copy, part_data, size);

	if (!message_part_serialize_update_header(part_data_copy, size,
						  hdr_size)) {
		t_pop();
		return FALSE;
	}

	index->update_field_raw(update, DATA_FIELD_MESSAGEPART,
				part_data_copy, size);
	t_pop();
	return TRUE;
}

static int match_next_record(MailIndex *index, MailIndexRecord *rec,
			     unsigned int seq, IStream *input,
			     MailIndexRecord **next_rec, int *dirty)
{
        MailIndexUpdate *update;
	MessageSize hdr_parsed_size;
	MboxHeaderContext ctx;
	uoff_t header_offset, body_offset, offset;
	uoff_t hdr_size, body_size;
	unsigned char current_digest[16];
	int hdr_size_fixed;

	*next_rec = NULL;

	/* skip the From-line */
	skip_line(input);
	header_offset = input->v_offset;

	hdr_size = 0; hdr_size_fixed = FALSE;
	do {
		if (!mbox_mail_get_location(index, rec, NULL, NULL, &body_size))
			return FALSE;

		i_stream_seek(input, header_offset);

		if (body_size == 0 && !hdr_size_fixed) {
			/* possibly broken message, find the next From-line
			   and make sure header parser won't pass it. */
			mbox_skip_header(input);
			i_stream_set_read_limit(input, input->v_offset);
			i_stream_seek(input, header_offset);
			hdr_size_fixed = TRUE;
			hdr_size = 0;
		}

		if (hdr_size == 0) {
			/* get the MD5 sum of fixed headers and the current
			   message flags in Status and X-Status fields */<commit_msg>fix warning with older gccs<commit_after>/* Copyright (C) 2002 <NAME> */

#include "lib.h"
#include "istream.h"
#include "hex-binary.h"
#include "message-parser.h"
#include "message-part-serialize.h"
#include "mbox-index.h"
#include "mbox-lock.h"
#include "mail-index-util.h"

#include <unistd.h>
#include <fcntl.h>

static void skip_line(IStream *input)
{
	const unsigned char *msg;
	size_t i, size;

	while (i_stream_read_data(input, &msg, &size, 0) > 0) {
		for (i = 0; i < size; i++) {
			if (msg[i] == '\n') {
				i_stream_skip(input, i+1);
				return;
			}
		}

		i_stream_skip(input, i);
	}
}

static int verify_header_md5sum(MailIndex *index, MailIndexRecord *rec,
				unsigned char current_digest[16])
{
	const unsigned char *old_digest;
	size_t size;

	/* MD5 sums must match */
	old_digest = index->lookup_field_raw(index, rec, DATA_FIELD_MD5, &size);
	return old_digest != NULL && size >= 16 &&
                memcmp(old_digest, current_digest, 16) == 0;
}

static int mail_update_header_size(MailIndex *index, MailIndexRecord *rec,
				   MailIndexUpdate *update,
				   MessageSize *hdr_size)
{
	const void *part_data;
	void *part_data_copy;
	uoff_t virtual_size;
	size_t size;

	/* update FIELD_HDR_HEADER_SIZE */
	index->update_field_raw(update, DATA_HDR_HEADER_SIZE,
				&hdr_size->physical_size,
				sizeof(hdr_size->physical_size));

	/* reset FIELD_HDR_VIRTUAL_SIZE - we don't know it anymore */
        virtual_size = (uoff_t)-1;
	index->update_field_raw(update, DATA_HDR_VIRTUAL_SIZE,
				&virtual_size, sizeof(virtual_size));

	/* update DATA_FIELD_MESSAGEPART */
	if ((rec->data_fields & DATA_FIELD_MESSAGEPART) == 0)
		return TRUE;

	part_data = index->lookup_field_raw(index, rec, DATA_FIELD_MESSAGEPART,
					    &size);
	if (part_data == NULL) {
		/* well, this wasn't expected but don't bother failing */
		return TRUE;
	}

	t_push();

	/* copy & update the part data */
	part_data_copy = t_malloc(size);
	memcpy(part_data_copy, part_data, size);

	if (!message_part_serialize_update_header(part_data_copy, size,
						  hdr_size)) {
		t_pop();
		return FALSE;
	}

	index->update_field_raw(update, DATA_FIELD_MESSAGEPART,
				part_data_copy, size);
	t_pop();
	return TRUE;
}

static int match_next_record(MailIndex *index, MailIndexRecord *rec,
			     unsigned int seq, IStream *input,
			     MailIndexRecord **next_rec, int *dirty)
{
        MailIndexUpdate *update;
	MessageSize hdr_parsed_size;
	MboxHeaderContext ctx;
	uoff_t header_offset, body_offset, offset;
	uoff_t hdr_size, body_size;
	unsigned char current_digest[16];
	int hdr_size_fixed;

	*next_rec = NULL;

	/* skip the From-line */
	skip_line(input);
	header_offset = input->v_offset;

	hdr_size = 0; body_offset = 0; hdr_size_fixed = FALSE;
	do {
		if (!mbox_mail_get_location(index, rec, NULL, NULL, &body_size))
			return FALSE;

		i_stream_seek(input, header_offset);

		if (body_size == 0 && !hdr_size_fixed) {
			/* possibly broken message, find the next From-line
			   and make sure header parser won't pass it. */
			mbox_skip_header(input);
			i_stream_set_read_limit(input, input->v_offset);
			i_stream_seek(input, header_offset);
			hdr_size_fixed = TRUE;
			hdr_size = 0;
		}

		if (hdr_size == 0) {
			/* get the MD5 sum of fixed headers and the current
			   message flags in Status and X-Status fields */
{'fraction_non_alphanumeric': 0.12047828061147269, 'fraction_numerical': 0.008324504313606781, 'mean_word_length': 4.098765432098766, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6870064', 'n_tokens_mistral': 2662, 'n_tokens_neox': 2383, 'n_words': 570}