<commit_before>/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "hash.h"
#include "mail-index-modseq.h"
#include "mail-storage-private.h"
#include "mail-search-build.h"
#include "dsync-transaction-log-scan.h"
#include "dsync-mail.h"
#include "dsync-mailbox-export.h"

struct dsync_mail_guid_instances {
	ARRAY_TYPE(seq_range) seqs;
	bool requested;
	bool searched;
};

struct dsync_mailbox_exporter {
	pool_t pool;
	struct mailbox *box;
	struct dsync_transaction_log_scan *log_scan;
	uint32_t last_common_uid;

	struct mailbox_transaction_context *trans;
	struct mail_search_context *search_ctx;

	/* GUID => instances */
	HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
	ARRAY_TYPE(seq_range) requested_uids;
	unsigned int requested_uid_search_idx;

	ARRAY_TYPE(seq_range) expunged_seqs;
	ARRAY_TYPE(const_string) expunged_guids;
	unsigned int expunged_guid_idx;

	/* uint32_t UID => struct dsync_mail_change */
	HASH_TABLE(void *, struct dsync_mail_change *) changes;
	/* changes sorted by UID */
	ARRAY(struct dsync_mail_change *) sorted_changes;
	unsigned int change_idx;
	uint32_t highest_changed_uid;

	struct dsync_mail_change change;
	struct dsync_mail dsync_mail;

	const char *error;
	unsigned int body_search_initialized:1;
	unsigned int auto_export_mails:1;
	unsigned int mails_have_guids:1;
	unsigned int return_all_mails:1;
};

static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
			    struct mail *mail, const char *field)
{
	const char *errstr;
	enum mail_error error;

	errstr = mailbox_get_last_error(exporter->box, &error);
	if (error == MAIL_ERROR_EXPUNGED)
		return 0;

	exporter->error = p_strdup_printf(exporter->pool,
		"Can't lookup %s for UID=%u: %s",
		field, mail->uid, errstr);
	return -1;
}

static bool
final_keyword_check(struct dsync_mail_change *change, const char *name)
{
	const char *const *changes;
	unsigned int i, count;

	changes = array_get(&change->keyword_changes, &count);
	for (i = 0; i < count; i++) {
		if (strcmp(changes[i]+1, name) == 0) {
			if (changes[i][0] == KEYWORD_CHANGE_REMOVE) {
				/* a final keyword is marked as removed.
				   this shouldn't normally happen. */
				array_delete(&change->keyword_changes, i, 1);
				break;
			}
			return TRUE;
		}
	}
	return FALSE;
}

static void
search_update_flag_changes(struct dsync_mailbox_exporter *exporter,
			   struct mail *mail, struct dsync_mail_change *change)
{
	const char *const *keywords;
	unsigned int i;

	i_assert((change->add_flags & change->remove_flags) == 0);

	change->modseq = mail_get_modseq(mail);
	change->pvt_modseq = mail_get_pvt_modseq(mail);
	change->final_flags = mail_get_flags(mail) & MAIL_FLAGS_NONRECENT;

	keywords = mail_get_keywords(mail);
	if (!array_is_created(&change->keyword_changes) &&
	    keywords[0] != NULL) {
		p_array_init(&change->keyword_changes, exporter->pool,
			     str_array_length(keywords));
	}
	for (i = 0; keywords[i] != NULL; i++) {
		/* add the final keyword if it's not already there
		   as +keyword */
		if (!final_keyword_check(change, keywords[i])) {
			const char *keyword_change =
				p_strdup_printf(exporter->pool, "%c%s",
						KEYWORD_CHANGE_FINAL,
						keywords[i]);
			array_append(&change->keyword_changes,
				     &keyword_change, 1);
		}
	}<commit_msg>dsync: Assert-crashfix for handling keywords.<commit_after>/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "hash.h"
#include "mail-index-modseq.h"
#include "mail-storage-private.h"
#include "mail-search-build.h"
#include "dsync-transaction-log-scan.h"
#include "dsync-mail.h"
#include "dsync-mailbox-export.h"

struct dsync_mail_guid_instances {
	ARRAY_TYPE(seq_range) seqs;
	bool requested;
	bool searched;
};

struct dsync_mailbox_exporter {
	pool_t pool;
	struct mailbox *box;
	struct dsync_transaction_log_scan *log_scan;
	uint32_t last_common_uid;

	struct mailbox_transaction_context *trans;
	struct mail_search_context *search_ctx;

	/* GUID => instances */
	HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
	ARRAY_TYPE(seq_range) requested_uids;
	unsigned int requested_uid_search_idx;

	ARRAY_TYPE(seq_range) expunged_seqs;
	ARRAY_TYPE(const_string) expunged_guids;
	unsigned int expunged_guid_idx;

	/* uint32_t UID => struct dsync_mail_change */
	HASH_TABLE(void *, struct dsync_mail_change *) changes;
	/* changes sorted by UID */
	ARRAY(struct dsync_mail_change *) sorted_changes;
	unsigned int change_idx;
	uint32_t highest_changed_uid;

	struct dsync_mail_change change;
	struct dsync_mail dsync_mail;

	const char *error;
	unsigned int body_search_initialized:1;
	unsigned int auto_export_mails:1;
	unsigned int mails_have_guids:1;
	unsigned int return_all_mails:1;
};

static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
			    struct mail *mail, const char *field)
{
	const char *errstr;
	enum mail_error error;

	errstr = mailbox_get_last_error(exporter->box, &error);
	if (error == MAIL_ERROR_EXPUNGED)
		return 0;

	exporter->error = p_strdup_printf(exporter->pool,
		"Can't lookup %s for UID=%u: %s",
		field, mail->uid, errstr);
	return -1;
}

static bool
final_keyword_check(struct dsync_mail_change *change, const char *name)
{
	const char *const *changes;
	unsigned int i, count;

	changes = array_get(&change->keyword_changes, &count);
	for (i = 0; i < count; i++) {
		if (strcmp(changes[i]+1, name) == 0) {
			if (changes[i][0] == KEYWORD_CHANGE_REMOVE) {
				/* a final keyword is marked as removed.
				   this shouldn't normally happen. */
				array_delete(&change->keyword_changes, i, 1);
				break;
			}
			return TRUE;
		}
	}
	return FALSE;
}

static void
search_update_flag_changes(struct dsync_mailbox_exporter *exporter,
			   struct mail *mail, struct dsync_mail_change *change)
{
	const char *const *keywords;
	unsigned int i;

	i_assert((change->add_flags & change->remove_flags) == 0);

	change->modseq = mail_get_modseq(mail);
	change->pvt_modseq = mail_get_pvt_modseq(mail);
	change->final_flags = mail_get_flags(mail) & MAIL_FLAGS_NONRECENT;

	keywords = mail_get_keywords(mail);
	if (!array_is_created(&change->keyword_changes) &&
	    keywords[0] != NULL) {
		p_array_init(&change->keyword_changes, exporter->pool,
			     str_array_length(keywords));
	}
	for (i = 0; keywords[i] != NULL; i++) {
		/* add the final keyword if it's not already there
		   as +keyword */
		if (!final_keyword_check(change, keywords[i])) {
			const char *keyword_change =
				p_strdup_printf(exporter->pool, "%c%s",
						KEYWORD_CHANGE_ADD,
						keywords[i]);
			array_append(&change->keyword_changes,
				     &keyword_change, 1);
		}
	}
{'fraction_non_alphanumeric': 0.11509827278141752, 'fraction_numerical': 0.007444907683144729, 'mean_word_length': 4.9813000890471955, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2221313', 'n_tokens_mistral': 2576, 'n_tokens_neox': 2386, 'n_words': 544}