<commit_before>/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "istream.h"
#include "ostream.h"
#include "str.h"
#include "hash.h"
#include "mdbox-storage.h"
#include "mdbox-storage-rebuild.h"
#include "mdbox-file.h"
#include "mdbox-map.h"
#include "mdbox-sync.h"

#include <stdlib.h>
#include <dirent.h>

/*
   Altmoving works like:

   1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox
      code and map UID's alt-refcount is updated. It won't be written to disk.
   2. mdbox_purge() is called, which checks if map UID's refcount equals
      to its alt-refcount. If it does, it's moved to alt storage. Moving to
      primary storage is done if _ALT flag was removed from any message.
*/

enum mdbox_msg_action {
	MDBOX_MSG_ACTION_MOVE_TO_ALT = 1,
	MDBOX_MSG_ACTION_MOVE_FROM_ALT
};

struct mdbox_purge_context {
	pool_t pool;
	struct mdbox_storage *storage;

	uint32_t lowest_primary_file_id;
	/* list of file_ids that exist in primary storage. this list is looked
	   up while there is no locking, so it may not be accurate anymore by
	   the time it's used. */
	ARRAY_TYPE(seq_range) primary_file_ids;
	/* list of file_ids that we need to purge */
	ARRAY_TYPE(seq_range) purge_file_ids;

	/* map_uid => mdbox_msg_action */
	struct hash_table *altmoves;
	bool have_altmoves;

	struct mdbox_map_atomic_context *atomic;
	struct mdbox_map_append_context *append_ctx;
};

static int mdbox_map_file_msg_offset_cmp(const void *p1, const void *p2)
{
	const struct mdbox_map_file_msg *m1 = p1, *m2 = p2;

	if (m1->offset < m2->offset)
		return -1;
	else if (m1->offset > m2->offset)
		return 1;
	else
		return 0;
}

static int
mdbox_file_copy_metadata(struct dbox_file *file, struct ostream *output)
{
	struct dbox_metadata_header meta_hdr;
	const char *line;
	const unsigned char *data;
	size_t size;
	int ret;

	ret = i_stream_read_data(file->input, &data, &size,
				 sizeof(meta_hdr));
	if (ret <= 0) {
		i_assert(ret == -1);
		if (file->input->stream_errno == 0) {
			dbox_file_set_corrupted(file, "missing metadata");
			return 0;
		}
		mail_storage_set_critical(&file->storage->storage,
			"read(%s) failed: %m", file->cur_path);
		return -1;
	}

	memcpy(&meta_hdr, data, sizeof(meta_hdr));
	if (memcmp(meta_hdr.magic_post, DBOX_MAGIC_POST,
		   sizeof(meta_hdr.magic_post)) != 0) {
		dbox_file_set_corrupted(file, "invalid metadata magic");
		return 0;
	}
	i_stream_skip(file->input, sizeof(meta_hdr));
	if (output != NULL)
		o_stream_send(output, &meta_hdr, sizeof(meta_hdr));
	while ((line = i_stream_read_next_line(file->input)) != NULL) {
		if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
			/* end of metadata */
			break;
		}
		if (output != NULL) {
			o_stream_send_str(output, line);
			o_stream_send(output, "\n", 1);
		}
	}
	if (line == NULL) {
		dbox_file_set_corrupted(file, "missing end-of-metadata line");
		return 0;
	}
	if (output != NULL)
		o_stream_send(output, "\n", 1);
	return 1;
}

static bool
mdbox_purge_want_altpath(struct mdbox_purge_context *ctx, uint32_t map_uid)
{
	enum mdbox_msg_action action;
	void *value;

	if (!ctx->have_altmoves)
		return FALSE;

	value = hash_table_lookup(ctx->altmoves, POINTER_CAST(map_uid));
	if (value == NULL)
		return FALSE;

	action = POINTER_CAST_TO(value, enum mdbox_msg_action);
	return action == MDBOX_MSG_ACTION_MOVE_TO_ALT;
}

static int
mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file,
		     const struct mdbox_map_file_msg *msg)
{
	struct dbox_file_append_context *out_file_append;<commit_msg>mdbox: Use unlimited buffer size when copying metadata while purging.<commit_after>/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "istream.h"
#include "ostream.h"
#include "str.h"
#include "hash.h"
#include "mdbox-storage.h"
#include "mdbox-storage-rebuild.h"
#include "mdbox-file.h"
#include "mdbox-map.h"
#include "mdbox-sync.h"

#include <stdlib.h>
#include <dirent.h>

/*
   Altmoving works like:

   1. Message's DBOX_INDEX_FLAG_ALT flag is changed. This is caught by mdbox
      code and map UID's alt-refcount is updated. It won't be written to disk.
   2. mdbox_purge() is called, which checks if map UID's refcount equals
      to its alt-refcount. If it does, it's moved to alt storage. Moving to
      primary storage is done if _ALT flag was removed from any message.
*/

enum mdbox_msg_action {
	MDBOX_MSG_ACTION_MOVE_TO_ALT = 1,
	MDBOX_MSG_ACTION_MOVE_FROM_ALT
};

struct mdbox_purge_context {
	pool_t pool;
	struct mdbox_storage *storage;

	uint32_t lowest_primary_file_id;
	/* list of file_ids that exist in primary storage. this list is looked
	   up while there is no locking, so it may not be accurate anymore by
	   the time it's used. */
	ARRAY_TYPE(seq_range) primary_file_ids;
	/* list of file_ids that we need to purge */
	ARRAY_TYPE(seq_range) purge_file_ids;

	/* map_uid => mdbox_msg_action */
	struct hash_table *altmoves;
	bool have_altmoves;

	struct mdbox_map_atomic_context *atomic;
	struct mdbox_map_append_context *append_ctx;
};

static int mdbox_map_file_msg_offset_cmp(const void *p1, const void *p2)
{
	const struct mdbox_map_file_msg *m1 = p1, *m2 = p2;

	if (m1->offset < m2->offset)
		return -1;
	else if (m1->offset > m2->offset)
		return 1;
	else
		return 0;
}

static int
mdbox_file_copy_metadata(struct dbox_file *file, struct ostream *output)
{
	struct dbox_metadata_header meta_hdr;
	const char *line;
	const unsigned char *data;
	size_t size, buf_size;
	int ret;

	ret = i_stream_read_data(file->input, &data, &size,
				 sizeof(meta_hdr));
	if (ret <= 0) {
		i_assert(ret == -1);
		if (file->input->stream_errno == 0) {
			dbox_file_set_corrupted(file, "missing metadata");
			return 0;
		}
		mail_storage_set_critical(&file->storage->storage,
			"read(%s) failed: %m", file->cur_path);
		return -1;
	}

	memcpy(&meta_hdr, data, sizeof(meta_hdr));
	if (memcmp(meta_hdr.magic_post, DBOX_MAGIC_POST,
		   sizeof(meta_hdr.magic_post)) != 0) {
		dbox_file_set_corrupted(file, "invalid metadata magic");
		return 0;
	}
	i_stream_skip(file->input, sizeof(meta_hdr));
	if (output != NULL)
		o_stream_send(output, &meta_hdr, sizeof(meta_hdr));

	buf_size = i_stream_get_max_buffer_size(file->input);
	i_stream_set_max_buffer_size(file->input, 0);
	while ((line = i_stream_read_next_line(file->input)) != NULL) {
		if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
			/* end of metadata */
			break;
		}
		if (output != NULL) {
			o_stream_send_str(output, line);
			o_stream_send(output, "\n", 1);
		}
	}
	i_stream_set_max_buffer_size(file->input, buf_size);

	if (line == NULL) {
		dbox_file_set_corrupted(file, "missing end-of-metadata line");
		return 0;
	}
	if (output != NULL)
		o_stream_send(output, "\n", 1);
	return 1;
}

static bool
mdbox_purge_want_altpath(struct mdbox_purge_context *ctx, uint32_t map_uid)
{
	enum mdbox_msg_action action;
	void *value;

	if (!ctx->have_altmoves)
		return FALSE;

	value = hash_table_lookup(ctx->altmoves, POINTER_CAST(map_uid));
	if (value == NULL)
		return FALSE;

	action = POINTER_CAST_TO(value, enum mdbox_msg_action);
	return action == MDBOX_MSG_ACTION_MOVE_TO_ALT;
}

static int
mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file,
		     const struct mdbox_map_file_msg *msg)
{
	struct dbox_file_append_context *out_file_append;
{'fraction_non_alphanumeric': 0.11545698924731183, 'fraction_numerical': 0.011155913978494624, 'mean_word_length': 4.56544502617801, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3946341', 'n_tokens_mistral': 3025, 'n_tokens_neox': 2799, 'n_words': 717}