<commit_before>/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "fdatasync-path.h"
#include "hex-binary.h"
#include "hex-dec.h"
#include "str.h"
#include "istream.h"
#include "istream-crlf.h"
#include "ostream.h"
#include "write-full.h"
#include "index-mail.h"
#include "mail-copy.h"
#include "dbox-attachment.h"
#include "dbox-save.h"
#include "sdbox-storage.h"
#include "sdbox-file.h"
#include "sdbox-sync.h"

#include <stdlib.h>

struct sdbox_save_context {
	struct dbox_save_context ctx;

	struct sdbox_mailbox *mbox;
	struct sdbox_sync_context *sync_ctx;
	struct dbox_file_append_context *append_ctx;

	uint32_t first_saved_seq;
	ARRAY_DEFINE(files, struct dbox_file *);
};

struct dbox_file *
sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq)
{
	struct sdbox_save_context *ctx =
		(struct sdbox_save_context *)t->save_ctx;
	struct dbox_file *const *files;
	unsigned int count;

	i_assert(seq >= ctx->first_saved_seq);

	files = array_get(&ctx->files, &count);
	i_assert(count > 0);
	i_assert(seq - ctx->first_saved_seq < count);

	return files[seq - ctx->first_saved_seq];
}

struct mail_save_context *
sdbox_save_alloc(struct mailbox_transaction_context *t)
{
	struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)t->box;
	struct sdbox_save_context *ctx =
		(struct sdbox_save_context *)t->save_ctx;

	i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);

	if (ctx != NULL) {
		/* use the existing allocated structure */
		ctx->ctx.failed = FALSE;
		ctx->ctx.finished = FALSE;
		ctx->ctx.cur_file = NULL;
		ctx->ctx.dbox_output = NULL;
		return &ctx->ctx.ctx;
	}

	ctx = i_new(struct sdbox_save_context, 1);
	ctx->ctx.ctx.transaction = t;
	ctx->ctx.trans = t->itrans;
	ctx->mbox = mbox;
	i_array_init(&ctx->files, 32);
	t->save_ctx = &ctx->ctx.ctx;
	return t->save_ctx;
}

void sdbox_save_add_file(struct mail_save_context *_ctx, struct dbox_file *file)
{
	struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;

	if (ctx->first_saved_seq == 0)
		ctx->first_saved_seq = ctx->ctx.seq;
	array_append(&ctx->files, &file, 1);
}

int sdbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
{
	struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
	struct dbox_file *file;
	int ret;

	file = sdbox_file_create(ctx->mbox);
	ctx->append_ctx = dbox_file_append_init(file);
	ret = dbox_file_get_append_stream(ctx->append_ctx,
					  &ctx->ctx.dbox_output);
	if (ret <= 0) {
		i_assert(ret != 0);
		dbox_file_append_rollback(&ctx->append_ctx);
		dbox_file_unref(&file);
		ctx->ctx.failed = TRUE;
		return -1;
	}
	ctx->ctx.cur_file = file;
	dbox_save_begin(&ctx->ctx, input);

	sdbox_save_add_file(_ctx, file);
	return ctx->ctx.failed ? -1 : 0;
}

static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx,
					 struct dbox_file *file)
{<commit_msg>sdbox: Avoid using too many fds when copying messages. This happened at least with quota plugin.<commit_after>/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "fdatasync-path.h"
#include "hex-binary.h"
#include "hex-dec.h"
#include "str.h"
#include "istream.h"
#include "istream-crlf.h"
#include "ostream.h"
#include "write-full.h"
#include "index-mail.h"
#include "mail-copy.h"
#include "dbox-attachment.h"
#include "dbox-save.h"
#include "sdbox-storage.h"
#include "sdbox-file.h"
#include "sdbox-sync.h"

#include <stdlib.h>

struct sdbox_save_context {
	struct dbox_save_context ctx;

	struct sdbox_mailbox *mbox;
	struct sdbox_sync_context *sync_ctx;
	struct dbox_file_append_context *append_ctx;

	uint32_t first_saved_seq;
	ARRAY_DEFINE(files, struct dbox_file *);
};

struct dbox_file *
sdbox_save_file_get_file(struct mailbox_transaction_context *t, uint32_t seq)
{
	struct sdbox_save_context *ctx =
		(struct sdbox_save_context *)t->save_ctx;
	struct dbox_file *const *files;
	unsigned int count;

	i_assert(seq >= ctx->first_saved_seq);

	files = array_get(&ctx->files, &count);
	i_assert(count > 0);
	i_assert(seq - ctx->first_saved_seq < count);

	return files[seq - ctx->first_saved_seq];
}

struct mail_save_context *
sdbox_save_alloc(struct mailbox_transaction_context *t)
{
	struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)t->box;
	struct sdbox_save_context *ctx =
		(struct sdbox_save_context *)t->save_ctx;

	i_assert((t->flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);

	if (ctx != NULL) {
		/* use the existing allocated structure */
		ctx->ctx.failed = FALSE;
		ctx->ctx.finished = FALSE;
		ctx->ctx.cur_file = NULL;
		ctx->ctx.dbox_output = NULL;
		return &ctx->ctx.ctx;
	}

	ctx = i_new(struct sdbox_save_context, 1);
	ctx->ctx.ctx.transaction = t;
	ctx->ctx.trans = t->itrans;
	ctx->mbox = mbox;
	i_array_init(&ctx->files, 32);
	t->save_ctx = &ctx->ctx.ctx;
	return t->save_ctx;
}

void sdbox_save_add_file(struct mail_save_context *_ctx, struct dbox_file *file)
{
	struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
	struct dbox_file *const *files;
	unsigned int count;

	if (ctx->first_saved_seq == 0)
		ctx->first_saved_seq = ctx->ctx.seq;

	files = array_get(&ctx->files, &count);
	if (count > 0) {
		/* a plugin may leave a previously saved file open.
		   we'll close it here to avoid eating too many fds. */
		dbox_file_close(files[count-1]);
	}
	array_append(&ctx->files, &file, 1);
}

int sdbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
{
	struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx;
	struct dbox_file *file;
	int ret;

	file = sdbox_file_create(ctx->mbox);
	ctx->append_ctx = dbox_file_append_init(file);
	ret = dbox_file_get_append_stream(ctx->append_ctx,
					  &ctx->ctx.dbox_output);
	if (ret <= 0) {
		i_assert(ret != 0);
		dbox_file_append_rollback(&ctx->append_ctx);
		dbox_file_unref(&file);
		ctx->ctx.failed = TRUE;
		return -1;
	}
	ctx->ctx.cur_file = file;
	dbox_save_begin(&ctx->ctx, input);

	sdbox_save_add_file(_ctx, file);
	return ctx->ctx.failed ? -1 : 0;
}

static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx,
					 struct dbox_file *file)
{
{'fraction_non_alphanumeric': 0.13542847885954964, 'fraction_numerical': 0.008099789405475458, 'mean_word_length': 5.0648330058939095, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7651492', 'n_tokens_mistral': 2549, 'n_tokens_neox': 2372, 'n_words': 504}