<commit_before>/* Copyright (C) 2004 <NAME> */

#include "lib.h"
#include "ioloop.h"
#include "buffer.h"
#include "file-set-size.h"
#include "mmap-util.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-transaction-util.h"
#include "mail-cache-private.h"

#include <time.h>

static void
mail_index_header_update_counts(struct mail_index_header *hdr,
				uint8_t old_flags, uint8_t new_flags)
{
	if (((old_flags ^ new_flags) & MAIL_RECENT) != 0) {
		/* different recent-flag */
		if ((old_flags & MAIL_RECENT) == 0)
			hdr->recent_messages_count++;
		else if (--hdr->recent_messages_count == 0)
			hdr->first_recent_uid_lowwater = hdr->next_uid;
	}

	if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
		/* different seen-flag */
		if ((old_flags & MAIL_SEEN) != 0)
			hdr->seen_messages_count--;
		else if (++hdr->seen_messages_count == hdr->messages_count)
			hdr->first_unseen_uid_lowwater = hdr->next_uid;
	}

	if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
		/* different deleted-flag */
		if ((old_flags & MAIL_DELETED) == 0)
			hdr->deleted_messages_count++;
		else if (--hdr->deleted_messages_count == 0)
			hdr->first_deleted_uid_lowwater = hdr->next_uid;
	}
}

static void
mail_index_header_update_lowwaters(struct mail_index_header *hdr,
				   const struct mail_index_record *rec)
{
	if ((rec->flags & MAIL_RECENT) != 0 &&
	    rec->uid < hdr->first_recent_uid_lowwater)
		hdr->first_recent_uid_lowwater = rec->uid;
	if ((rec->flags & MAIL_SEEN) == 0 &&
	    rec->uid < hdr->first_unseen_uid_lowwater)
		hdr->first_unseen_uid_lowwater = rec->uid;
	if ((rec->flags & MAIL_DELETED) != 0 &&
	    rec->uid < hdr->first_deleted_uid_lowwater)
		hdr->first_deleted_uid_lowwater = rec->uid;
}

static void mail_index_sync_cache_expunge(struct mail_index_sync_map_ctx *ctx,
					  uoff_t cache_offset)
{
	if (!ctx->update_cache)
		return;

	if (!ctx->cache_locked) {
		if (mail_cache_lock(ctx->view->index->cache) <= 0)
			return;
		ctx->cache_locked = TRUE;
	}

	if (!MAIL_CACHE_IS_UNUSABLE(ctx->view->index->cache) &&
	    ctx->view->map->hdr->cache_file_seq !=
	    ctx->view->index->cache->hdr->file_seq)
		(void)mail_cache_delete(ctx->view->index->cache, cache_offset);
}

static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
{
        struct mail_index_sync_map_ctx *ctx = context;
	struct mail_index_view *view = ctx->view;
	struct mail_index_map *map = view->map;
	struct mail_index_header *hdr;
	struct mail_index_record *rec;
	uint32_t count, seq, seq1, seq2;

	i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));

	if (mail_index_lookup_uid_range(view, e->uid1, e->uid2,
					&seq1, &seq2) < 0)
		return -1;

	if (seq1 == 0)
		return 1;

	hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
	for (seq = seq1; seq <= seq2; seq++) {
                rec = MAIL_INDEX_MAP_IDX(map, seq-1);
		mail_index_header_update_counts(hdr, rec->flags, 0);
		
		if (rec->cache_offset != 0)
			mail_index_sync_cache_expunge(ctx, rec->cache_offset);
	}

	/* @UNSAFE */
	count = seq2 - seq1 + 1;
	memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2),
		(map->records_count - seq2) * map->hdr->record_size);

	map->records_count -= count;
	hdr->messages_count -= count;
	view->messages_count -= count;

	if (map->buffer != NULL) {
		buffer_set_used_size(map->buffer, map->records_count *
				     map->hdr->record_size);
		map->records = buffer_get_modifyable_data(map->buffer, NULL);
	}
	return 1;
}

static int sync_append(const struct mail_index_record *rec, void *context)
{
        struct mail_index_sync_map_ctx *ctx = context;
	struct mail_index_view *view = ctx->view;
	struct mail_index_map *map = view->map;
	struct mail_index_header *hdr;
	void *dest;

	hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
	if (rec->uid < hdr->next_uid) {
		mail_transaction_log_view_set_corrupted(view->log_view,
			"Append with UID %u, but next_uid = %u",
			rec->uid, hdr->next_uid);
		return -1;
	}

	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
		i_assert(map->records_count * map->hdr->record_size ==
			 buffer_get_used_size(map->buffer));
		dest = buffer_append_space_unsafe(map->buffer,
						  map->hdr->record_size);
		map->records = buffer_get_modifyable_data(map->buffer, NULL);
	} else {
		i_assert((map->records_count+1) * map->hdr->record_size <=
			 map->mmap_size);
		dest = MAIL_INDEX_MAP_IDX(map, map->records_count);
	}
	memcpy(dest, rec, sizeof(*rec));
	memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
	       map->hdr->record_size - sizeof(*rec));

	hdr->messages_count++;
	hdr->next_uid = rec->uid+1;
	view->messages_count++;
	map->records_count++;

	mail_index_header_update_counts(hdr, 0, rec->flags);
	mail_index_header_update_lowwaters(hdr, rec);
	return 1;
}

static int sync_flag_update(const struct mail_transaction_flag_update *u,
			    void *context)
{
        struct mail_index_sync_map_ctx *ctx = context;
	struct mail_index_view *view = ctx->view;
	struct mail_index_header *hdr;
	struct mail_index_record *rec;
	uint8_t flag_mask, old_flags;
	keywords_mask_t keyword_mask;
	uint32_t i, idx, seq1, seq2;
	int update_keywords;

	if (mail_index_lookup_uid_range(view, u->uid1, u->uid2,
					&seq1, &seq2) < 0)
		return -1;

	if (seq1 == 0)<commit_msg>hdr_flag_have_dirty wasn't set if dirty flags were set in appends.<commit_after>/* Copyright (C) 2004 <NAME> */

#include "lib.h"
#include "ioloop.h"
#include "buffer.h"
#include "file-set-size.h"
#include "mmap-util.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"
#include "mail-transaction-util.h"
#include "mail-cache-private.h"

#include <time.h>

static void
mail_index_header_update_counts(struct mail_index_header *hdr,
				uint8_t old_flags, uint8_t new_flags)
{
	if (((old_flags ^ new_flags) & MAIL_RECENT) != 0) {
		/* different recent-flag */
		if ((old_flags & MAIL_RECENT) == 0)
			hdr->recent_messages_count++;
		else if (--hdr->recent_messages_count == 0)
			hdr->first_recent_uid_lowwater = hdr->next_uid;
	}

	if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) {
		/* different seen-flag */
		if ((old_flags & MAIL_SEEN) != 0)
			hdr->seen_messages_count--;
		else if (++hdr->seen_messages_count == hdr->messages_count)
			hdr->first_unseen_uid_lowwater = hdr->next_uid;
	}

	if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) {
		/* different deleted-flag */
		if ((old_flags & MAIL_DELETED) == 0)
			hdr->deleted_messages_count++;
		else if (--hdr->deleted_messages_count == 0)
			hdr->first_deleted_uid_lowwater = hdr->next_uid;
	}
}

static void
mail_index_header_update_lowwaters(struct mail_index_header *hdr,
				   const struct mail_index_record *rec)
{
	if ((rec->flags & MAIL_RECENT) != 0 &&
	    rec->uid < hdr->first_recent_uid_lowwater)
		hdr->first_recent_uid_lowwater = rec->uid;
	if ((rec->flags & MAIL_SEEN) == 0 &&
	    rec->uid < hdr->first_unseen_uid_lowwater)
		hdr->first_unseen_uid_lowwater = rec->uid;
	if ((rec->flags & MAIL_DELETED) != 0 &&
	    rec->uid < hdr->first_deleted_uid_lowwater)
		hdr->first_deleted_uid_lowwater = rec->uid;
}

static void mail_index_sync_cache_expunge(struct mail_index_sync_map_ctx *ctx,
					  uoff_t cache_offset)
{
	if (!ctx->update_cache)
		return;

	if (!ctx->cache_locked) {
		if (mail_cache_lock(ctx->view->index->cache) <= 0)
			return;
		ctx->cache_locked = TRUE;
	}

	if (!MAIL_CACHE_IS_UNUSABLE(ctx->view->index->cache) &&
	    ctx->view->map->hdr->cache_file_seq !=
	    ctx->view->index->cache->hdr->file_seq)
		(void)mail_cache_delete(ctx->view->index->cache, cache_offset);
}

static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
{
        struct mail_index_sync_map_ctx *ctx = context;
	struct mail_index_view *view = ctx->view;
	struct mail_index_map *map = view->map;
	struct mail_index_header *hdr;
	struct mail_index_record *rec;
	uint32_t count, seq, seq1, seq2;

	i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));

	if (mail_index_lookup_uid_range(view, e->uid1, e->uid2,
					&seq1, &seq2) < 0)
		return -1;

	if (seq1 == 0)
		return 1;

	hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
	for (seq = seq1; seq <= seq2; seq++) {
                rec = MAIL_INDEX_MAP_IDX(map, seq-1);
		mail_index_header_update_counts(hdr, rec->flags, 0);
		
		if (rec->cache_offset != 0)
			mail_index_sync_cache_expunge(ctx, rec->cache_offset);
	}

	/* @UNSAFE */
	count = seq2 - seq1 + 1;
	memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2),
		(map->records_count - seq2) * map->hdr->record_size);

	map->records_count -= count;
	hdr->messages_count -= count;
	view->messages_count -= count;

	if (map->buffer != NULL) {
		buffer_set_used_size(map->buffer, map->records_count *
				     map->hdr->record_size);
		map->records = buffer_get_modifyable_data(map->buffer, NULL);
	}
	return 1;
}

static int sync_append(const struct mail_index_record *rec, void *context)
{
        struct mail_index_sync_map_ctx *ctx = context;
	struct mail_index_view *view = ctx->view;
	struct mail_index_map *map = view->map;
	struct mail_index_header *hdr;
	void *dest;

	hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL);
	if (rec->uid < hdr->next_uid) {
		mail_transaction_log_view_set_corrupted(view->log_view,
			"Append with UID %u, but next_uid = %u",
			rec->uid, hdr->next_uid);
		return -1;
	}

	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
		i_assert(map->records_count * map->hdr->record_size ==
			 buffer_get_used_size(map->buffer));
		dest = buffer_append_space_unsafe(map->buffer,
						  map->hdr->record_size);
		map->records = buffer_get_modifyable_data(map->buffer, NULL);
	} else {
		i_assert((map->records_count+1) * map->hdr->record_size <=
			 map->mmap_size);
		dest = MAIL_INDEX_MAP_IDX(map, map->records_count);
	}
	memcpy(dest, rec, sizeof(*rec));
	memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
	       map->hdr->record_size - sizeof(*rec));

	hdr->messages_count++;
	hdr->next_uid = rec->uid+1;
	view->messages_count++;
	map->records_count++;

	if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
		hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;

	mail_index_header_update_counts(hdr, 0, rec->flags);
	mail_index_header_update_lowwaters(hdr, rec);
	return 1;
}

static int sync_flag_update(const struct mail_transaction_flag_update *u,
			    void *context)
{
        struct mail_index_sync_map_ctx *ctx = context;
	struct mail_index_view *view = ctx->view;
	struct mail_index_header *hdr;
	struct mail_index_record *rec;
	uint8_t flag_mask, old_flags;
	keywords_mask_t keyword_mask;
	uint32_t i, idx, seq1, seq2;
	int update_keywords;

	if (mail_index_lookup_uid_range(view, u->uid1, u->uid2,
					&seq1, &seq2) < 0)
		return -1;

	if (seq1 == 0)
{'fraction_non_alphanumeric': 0.14542407597058002, 'fraction_numerical': 0.011823852527697607, 'mean_word_length': 4.525720164609053, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 215, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7644961', 'n_tokens_mistral': 4423, 'n_tokens_neox': 4123, 'n_words': 752}