<commit_before>/* Copyright (C) 2005 <NAME> */

#include "lib.h"
#include "ioloop.h"
#include "hex-dec.h"
#include "write-full.h"
#include "ostream.h"
#include "dbox-uidlist.h"
#include "dbox-sync.h"
#include "dbox-storage.h"

#include <stddef.h>

struct dbox_save_context {
	struct mail_save_context ctx;

	struct dbox_mailbox *mbox;
	struct mail_index_transaction *trans;
	struct dbox_uidlist_append_ctx *append_ctx;

	uint32_t first_append_seq;
	struct mail_index_sync_ctx *index_sync_ctx;

	/* updated for each appended mail: */
	uint32_t seq;
	struct istream *input;
	struct ostream *output;
        struct dbox_file *file;
	uint64_t hdr_offset;
	uint64_t mail_offset;

	bool failed;
};

struct mail_save_context *
dbox_save_init(struct mailbox_transaction_context *_t,
	       enum mail_flags flags, struct mail_keywords *keywords,
	       time_t received_date, int timezone_offset __attr_unused__,
	       const char *from_envelope __attr_unused__,
	       struct istream *input, bool want_mail __attr_unused__)
{
	struct dbox_transaction_context *t =
		(struct dbox_transaction_context *)_t;
	struct dbox_mailbox *mbox = (struct dbox_mailbox *)t->ictx.ibox;
	struct dbox_save_context *ctx = t->save_ctx;
	struct dbox_mail_header hdr;
	enum mail_flags save_flags;
	unsigned int left;
	char buf[128];
	int ret;

	i_assert((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);

	if (received_date == (time_t)-1)
		received_date = ioloop_time;

	if (ctx == NULL) {
		ctx = t->save_ctx = i_new(struct dbox_save_context, 1);
		ctx->ctx.transaction = &t->ictx.mailbox_ctx;
		ctx->mbox = mbox;
		ctx->trans = t->ictx.trans;
		ctx->append_ctx = dbox_uidlist_append_init(mbox->uidlist);

		if ((ret = dbox_sync_if_changed(mbox)) < 0) {
			ctx->failed = TRUE;
			return &ctx->ctx;
		}
		if (ret > 0) {
			if (dbox_sync(mbox, FALSE) < 0) {
				ctx->failed = TRUE;
				return &ctx->ctx;
			}
		}
	}
	ctx->input = input;

	if (dbox_uidlist_append_locked(ctx->append_ctx, &ctx->file) < 0) {
		ctx->failed = TRUE;
		return &ctx->ctx;
	}
	ctx->hdr_offset = ctx->file->output->offset;

	/* append mail header. UID and mail size are written later. */
	memset(&hdr, '0', sizeof(hdr));
	memcpy(hdr.magic, DBOX_MAIL_HEADER_MAGIC, sizeof(hdr.magic));
	DEC2HEX(hdr.received_time_hex, received_date);
	hdr.answered = (flags & MAIL_ANSWERED) != 0 ? '1' : '0';
	hdr.flagged = (flags & MAIL_FLAGGED) != 0 ? '1' : '0';
	hdr.deleted = (flags & MAIL_DELETED) != 0 ? '1' : '0';
	hdr.seen = (flags & MAIL_SEEN) != 0 ? '1' : '0';
	hdr.draft = (flags & MAIL_DRAFT) != 0 ? '1' : '0';
	hdr.expunged = '0';
	// FIXME: keywords
	o_stream_send(ctx->file->output, &hdr, sizeof(hdr));

	/* write rest of the header with '0' characters */
	left = ctx->file->mail_header_size - sizeof(hdr);
	memset(buf, '0', sizeof(buf));
	while (left > sizeof(buf)) {
		o_stream_send(ctx->file->output, buf, sizeof(buf));
		left -= sizeof(buf);
	}
	o_stream_send(ctx->file->output, buf, left);
	ctx->mail_offset = ctx->file->output->offset;

	/* add to index */
	save_flags = (flags & ~MAIL_RECENT) | MAIL_RECENT;
	mail_index_append(ctx->trans, 0, &ctx->seq);
	mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
				save_flags);
	if (keywords != NULL) {
		mail_index_update_keywords(ctx->trans, ctx->seq,
					   MODIFY_REPLACE, keywords);
	}
	mail_index_update_ext(ctx->trans, ctx->seq, mbox->dbox_file_ext_idx,
			      &ctx->file->file_seq, NULL);
	mail_index_update_ext(ctx->trans, ctx->seq,
			      mbox->dbox_offset_ext_idx, &ctx->hdr_offset,
			      NULL);

	if (ctx->first_append_seq == 0)
		ctx->first_append_seq = ctx->seq;
	return &ctx->ctx;
}

int dbox_save_continue(struct mail_save_context *_ctx)
{
	struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;

	if (ctx->failed)
		return -1;

	if (o_stream_send_istream(ctx->file->output, ctx->input) < 0) {
		if (ENOSPACE(ctx->file->output->stream_errno)) {
			mail_storage_set_error(STORAGE(ctx->mbox->storage),
					       "Not enough disk space");
		} else {
			mail_storage_set_critical(STORAGE(ctx->mbox->storage),
				"o_stream_send_istream(%s) failed: %m",<commit_msg>Write keywords when appending mails.<commit_after>/* Copyright (C) 2005 <NAME> */

#include "lib.h"
#include "array.h"
#include "ioloop.h"
#include "hex-dec.h"
#include "write-full.h"
#include "ostream.h"
#include "seq-range-array.h"
#include "dbox-uidlist.h"
#include "dbox-keywords.h"
#include "dbox-sync.h"
#include "dbox-storage.h"

#include <stddef.h>

struct dbox_save_context {
	struct mail_save_context ctx;

	struct dbox_mailbox *mbox;
	struct mail_index_transaction *trans;
	struct dbox_uidlist_append_ctx *append_ctx;

	uint32_t first_append_seq;
	struct mail_index_sync_ctx *index_sync_ctx;

	/* updated for each appended mail: */
	uint32_t seq;
	struct istream *input;
	struct ostream *output;
        struct dbox_file *file;
	uint64_t hdr_offset;
	uint64_t mail_offset;

	bool failed;
};

static int
dbox_save_add_keywords(struct dbox_save_context *ctx,
		       const struct mail_keywords *keywords,
		       buffer_t *file_keywords)
{
	array_t ARRAY_DEFINE(new_keywords, struct seq_range);
	const struct seq_range *range;
	unsigned int i, count, file_idx;
	int ret = 0;

	/* Get a list of all new keywords. Using seq_range is the easiest
	   way to do this and should be pretty fast too. */
	t_push();
	ARRAY_CREATE(&new_keywords, pool_datastack_create(),
		     struct seq_range, 16);
	for (i = 0; i < keywords->count; i++) {
		/* check if it's already in the file */
		if (dbox_file_lookup_keyword(ctx->mbox, ctx->file,
					     keywords->idx[i], &file_idx)) {
			buffer_write(file_keywords, file_idx, "1", 1);
			continue;
		}

		/* add it. if it already exists, it's handled internally. */
		seq_range_array_add(&new_keywords, 0, keywords->idx[i]);
	}

	/* now, write them to file */
	range = array_get(&new_keywords, &count);
	if (count > 0) {
		if (dbox_file_append_keywords(ctx->mbox, ctx->file,
					      range, count) < 0) {
			ret = -1;
			count = 0;
		}

		/* write the new keywords to file_keywords */
		for (i = 0; i < count; i++) {
			unsigned int kw;

			for (kw = range[i].seq1; kw <= range[i].seq2; kw++) {
				if (!dbox_file_lookup_keyword(ctx->mbox,
							      ctx->file, kw,
							      &file_idx)) {
					/* it should have been found */
					i_unreached();
					continue;
				}

				buffer_write(file_keywords, file_idx, "1", 1);
			}
		}
	}

	t_pop();
	return ret;
}

struct mail_save_context *
dbox_save_init(struct mailbox_transaction_context *_t,
	       enum mail_flags flags, struct mail_keywords *keywords,
	       time_t received_date, int timezone_offset __attr_unused__,
	       const char *from_envelope __attr_unused__,
	       struct istream *input, bool want_mail __attr_unused__)
{
	struct dbox_transaction_context *t =
		(struct dbox_transaction_context *)_t;
	struct dbox_mailbox *mbox = (struct dbox_mailbox *)t->ictx.ibox;
	struct dbox_save_context *ctx = t->save_ctx;
	struct dbox_mail_header hdr;
	buffer_t *file_keywords = NULL;
	enum mail_flags save_flags;
	unsigned int i, pos, left;
	char buf[128];
	int ret;

	i_assert((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);

	if (received_date == (time_t)-1)
		received_date = ioloop_time;

	if (ctx == NULL) {
		ctx = t->save_ctx = i_new(struct dbox_save_context, 1);
		ctx->ctx.transaction = &t->ictx.mailbox_ctx;
		ctx->mbox = mbox;
		ctx->trans = t->ictx.trans;
		ctx->append_ctx = dbox_uidlist_append_init(mbox->uidlist);

		if ((ret = dbox_sync_if_changed(mbox)) < 0) {
			ctx->failed = TRUE;
			return &ctx->ctx;
		}
		if (ret > 0) {
			if (dbox_sync(mbox, FALSE) < 0) {
				ctx->failed = TRUE;
				return &ctx->ctx;
			}
		}
	}
	ctx->input = input;

	if (dbox_uidlist_append_locked(ctx->append_ctx, &ctx->file) < 0) {
		ctx->failed = TRUE;
		return &ctx->ctx;
	}
	ctx->hdr_offset = ctx->file->output->offset;

	t_push();
	if (keywords != NULL && keywords->count > 0) {
		/* write keywords to the file */
		file_keywords = buffer_create_dynamic(pool_datastack_create(),
						      DBOX_KEYWORD_COUNT);
		if (dbox_save_add_keywords(ctx, keywords, file_keywords) < 0) {
			ctx->failed = TRUE;
			t_pop();
			return &ctx->ctx;
		}
		o_stream_seek(ctx->file->output, ctx->hdr_offset);
	}

	/* append mail header. UID and mail size are written later. */
	memset(&hdr, '0', sizeof(hdr));
	memcpy(hdr.magic, DBOX_MAIL_HEADER_MAGIC, sizeof(hdr.magic));
	DEC2HEX(hdr.received_time_hex, received_date);
	hdr.answered = (flags & MAIL_ANSWERED) != 0 ? '1' : '0';
	hdr.flagged = (flags & MAIL_FLAGGED) != 0 ? '1' : '0';
	hdr.deleted = (flags & MAIL_DELETED) != 0 ? '1' : '0';
	hdr.seen = (flags & MAIL_SEEN) != 0 ? '1' : '0';
	hdr.draft = (flags & MAIL_DRAFT) != 0 ? '1' : '0';
	hdr.expunged = '0';
	o_stream_send(ctx->file->output, &hdr, sizeof(hdr));

	/* write keywords */
	if (file_keywords != NULL) {
		unsigned char *keyword_string;
		size_t size;

		keyword_string =
			buffer_get_modifyable_data(file_keywords, &size);

		/* string should be filled with NULs and '1' now.
		   Change NULs to '0'. */
		for (i = 0; i < size; i++) {
			if (keyword_string[i] == '\0')
				keyword_string[i] = '0';
		}
		o_stream_send(ctx->file->output, keyword_string, size);
	}

	/* fill rest of the header with '0' characters */
	pos = ctx->file->output->offset - ctx->hdr_offset;
	i_assert(pos <= ctx->file->mail_header_size);
	left = ctx->file->mail_header_size - pos;
	memset(buf, '0', I_MIN(sizeof(buf), left));
	while (left > sizeof(buf)) {
		o_stream_send(ctx->file->output, buf, sizeof(buf));
		left -= sizeof(buf);
	}
	o_stream_send(ctx->file->output, buf, left);
	ctx->mail_offset = ctx->file->output->offset;

	/* add to index */
	save_flags = (flags & ~MAIL_RECENT) | MAIL_RECENT;
	mail_index_append(ctx->trans, 0, &ctx->seq);
	mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
				save_flags);
	if (keywords != NULL) {
		mail_index_update_keywords(ctx->trans, ctx->seq,
					   MODIFY_REPLACE, keywords);
	}
	mail_index_update_ext(ctx->trans, ctx->seq, mbox->dbox_file_ext_idx,
			      &ctx->file->file_seq, NULL);
	mail_index_update_ext(ctx->trans, ctx->seq,
			      mbox->dbox_offset_ext_idx, &ctx->hdr_offset,
			      NULL);

	if (ctx->first_append_seq == 0)
		ctx->first_append_seq = ctx->seq;
	t_pop();
	return &ctx->ctx;
}

int dbox_save_continue(struct mail_save_context *_ctx)
{
	struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;

	if (ctx->failed)
		return -1;

	if (o_stream_send_istream(ctx->file->output, ctx->input) < 0) {
		if (ENOSPACE(ctx->file->output->stream_errno)) {
			mail_storage_set_error(STORAGE(ctx->mbox->storage),
					       "Not enough disk space");
		} else {
			mail_storage_set_critical(STORAGE(ctx->mbox->storage),
				"o_stream_send_istream(%s) failed: %m",
{'fraction_non_alphanumeric': 0.14755927475592748, 'fraction_numerical': 0.010692701069270108, 'mean_word_length': 3.9981412639405205, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 168, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5093261', 'n_tokens_mistral': 4454, 'n_tokens_neox': 4072, 'n_words': 884}