<commit_before>/* Copyright (C) 2004 <NAME> */

#include "lib.h"
#include "array.h"
#include "buffer.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"

static int
keyword_lookup(struct mail_index_sync_map_ctx *ctx,
	       const char *keyword_name, unsigned int *idx_r)
{
	struct mail_index_map *map = ctx->view->map;
	const unsigned int *idx_map;
	unsigned int i, count, keyword_idx;

	if (!map->keywords_read) {
		if (mail_index_map_parse_keywords(ctx->view->index, map) < 0)
			return -1;
	}
	if (array_is_created(&map->keyword_idx_map) &&
	    mail_index_keyword_lookup(ctx->view->index, keyword_name,
				      FALSE, &keyword_idx)) {
		/* FIXME: slow. maybe create index -> file mapping as well */
		idx_map = array_get(&map->keyword_idx_map, &count);
		for (i = 0; i < count; i++) {
			if (idx_map[i] == keyword_idx) {
				*idx_r = i;
				return 1;
			}
		}
	}

	*idx_r = (unsigned int)-1;
	return 0;
}

static buffer_t *
keywords_get_header_buf(struct mail_index_map *map,
			const struct mail_index_ext *ext,
			unsigned int new_count, unsigned int *keywords_count_r,
			size_t *rec_offset_r, size_t *name_offset_root_r,
			size_t *name_offset_r)
{
	buffer_t *buf;
	const struct mail_index_keyword_header *kw_hdr;
	const struct mail_index_keyword_header_rec *kw_rec;
	const char *name;
	struct mail_index_keyword_header new_kw_hdr;
	uint32_t offset;

	kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
	kw_rec = (const void *)(kw_hdr + 1);
	name = (const char *)(kw_rec + kw_hdr->keywords_count);

	if (kw_hdr->keywords_count == 0)
		return NULL;

	new_kw_hdr = *kw_hdr;
	new_kw_hdr.keywords_count += new_count;
	*keywords_count_r = new_kw_hdr.keywords_count;

	offset = kw_rec[kw_hdr->keywords_count-1].name_offset;
	offset += strlen(name + offset) + 1;

	buf = buffer_create_dynamic(pool_datastack_create(), 512);
	buffer_append(buf, &new_kw_hdr, sizeof(new_kw_hdr));
	buffer_append(buf, kw_rec, sizeof(*kw_rec) * kw_hdr->keywords_count);
	*rec_offset_r = buf->used;
	buffer_write(buf, buf->used + sizeof(*kw_rec) * new_count,
		     name, offset);
	*name_offset_root_r = buf->used;
	*name_offset_r = offset;
	return buf;
}

static int keywords_ext_register(struct mail_index_sync_map_ctx *ctx,
				 uint32_t ext_id, uint32_t reset_id,
				 uint32_t hdr_size, uint32_t keywords_count)
{
	buffer_t *ext_intro_buf;
	struct mail_transaction_ext_intro *u;

	ext_intro_buf =
		buffer_create_static_hard(pool_datastack_create(),
					  sizeof(*u) + sizeof("keywords")-1);

	u = buffer_append_space_unsafe(ext_intro_buf, sizeof(*u));
	u->ext_id = ext_id;
	u->reset_id = reset_id;
	u->hdr_size = hdr_size;
	u->record_size = (keywords_count + CHAR_BIT - 1) / CHAR_BIT;
	if ((u->record_size % 4) != 0) {
		/* since we aren't properly aligned anyway,
		   reserve one extra byte for future */
		u->record_size++;
	}
	u->record_align = 1;

	if (ext_id == (uint32_t)-1) {
		u->name_size = strlen("keywords");
		buffer_append(ext_intro_buf, "keywords", u->name_size);
	}

	return mail_index_sync_ext_intro(ctx, u);
}

static int
keywords_header_add(struct mail_index_sync_map_ctx *ctx,
		    const char *keyword_name, unsigned int *keyword_idx_r)
{
	struct mail_index_map *map = ctx->view->map;
        const struct mail_index_ext *ext = NULL;
	struct mail_index_keyword_header *kw_hdr;
	struct mail_index_keyword_header_rec kw_rec;
	uint32_t ext_id;
	buffer_t *buf = NULL;
	size_t keyword_len, rec_offset, name_offset, name_offset_root;
	unsigned int keywords_count;
	int ret;

	ext_id = mail_index_map_lookup_ext(map, "keywords");
	if (ext_id != (uint32_t)-1) {
		/* update existing header */
		ext = array_idx(&map->extensions, ext_id);<commit_msg>When adding new keywords, update the index file atomically so the keywords won't break if we crash in the middle of the header update.<commit_after>/* Copyright (C) 2004 <NAME> */

#include "lib.h"
#include "array.h"
#include "buffer.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"

static int
keyword_lookup(struct mail_index_sync_map_ctx *ctx,
	       const char *keyword_name, unsigned int *idx_r)
{
	struct mail_index_map *map = ctx->view->map;
	const unsigned int *idx_map;
	unsigned int i, count, keyword_idx;

	if (!map->keywords_read) {
		if (mail_index_map_parse_keywords(ctx->view->index, map) < 0)
			return -1;
	}
	if (array_is_created(&map->keyword_idx_map) &&
	    mail_index_keyword_lookup(ctx->view->index, keyword_name,
				      FALSE, &keyword_idx)) {
		/* FIXME: slow. maybe create index -> file mapping as well */
		idx_map = array_get(&map->keyword_idx_map, &count);
		for (i = 0; i < count; i++) {
			if (idx_map[i] == keyword_idx) {
				*idx_r = i;
				return 1;
			}
		}
	}

	*idx_r = (unsigned int)-1;
	return 0;
}

static buffer_t *
keywords_get_header_buf(struct mail_index_map *map,
			const struct mail_index_ext *ext,
			unsigned int new_count, unsigned int *keywords_count_r,
			size_t *rec_offset_r, size_t *name_offset_root_r,
			size_t *name_offset_r)
{
	buffer_t *buf;
	const struct mail_index_keyword_header *kw_hdr;
	const struct mail_index_keyword_header_rec *kw_rec;
	const char *name;
	struct mail_index_keyword_header new_kw_hdr;
	uint32_t offset;

	kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
	kw_rec = (const void *)(kw_hdr + 1);
	name = (const char *)(kw_rec + kw_hdr->keywords_count);

	if (kw_hdr->keywords_count == 0)
		return NULL;

	new_kw_hdr = *kw_hdr;
	new_kw_hdr.keywords_count += new_count;
	*keywords_count_r = new_kw_hdr.keywords_count;

	offset = kw_rec[kw_hdr->keywords_count-1].name_offset;
	offset += strlen(name + offset) + 1;

	buf = buffer_create_dynamic(pool_datastack_create(), 512);
	buffer_append(buf, &new_kw_hdr, sizeof(new_kw_hdr));
	buffer_append(buf, kw_rec, sizeof(*kw_rec) * kw_hdr->keywords_count);
	*rec_offset_r = buf->used;
	buffer_write(buf, buf->used + sizeof(*kw_rec) * new_count,
		     name, offset);
	*name_offset_root_r = buf->used;
	*name_offset_r = offset;
	return buf;
}

static int keywords_ext_register(struct mail_index_sync_map_ctx *ctx,
				 uint32_t ext_id, uint32_t reset_id,
				 uint32_t hdr_size, uint32_t keywords_count)
{
	buffer_t *ext_intro_buf;
	struct mail_transaction_ext_intro *u;

	ext_intro_buf =
		buffer_create_static_hard(pool_datastack_create(),
					  sizeof(*u) + sizeof("keywords")-1);

	u = buffer_append_space_unsafe(ext_intro_buf, sizeof(*u));
	u->ext_id = ext_id;
	u->reset_id = reset_id;
	u->hdr_size = hdr_size;
	u->record_size = (keywords_count + CHAR_BIT - 1) / CHAR_BIT;
	if ((u->record_size % 4) != 0) {
		/* since we aren't properly aligned anyway,
		   reserve one extra byte for future */
		u->record_size++;
	}
	u->record_align = 1;

	if (ext_id == (uint32_t)-1) {
		u->name_size = strlen("keywords");
		buffer_append(ext_intro_buf, "keywords", u->name_size);
	}

	return mail_index_sync_ext_intro(ctx, u);
}

static int
keywords_header_add(struct mail_index_sync_map_ctx *ctx,
		    const char *keyword_name, unsigned int *keyword_idx_r)
{
	struct mail_index_map *map = ctx->view->map;
        const struct mail_index_ext *ext = NULL;
	struct mail_index_keyword_header *kw_hdr;
	struct mail_index_keyword_header_rec kw_rec;
	uint32_t ext_id;
	buffer_t *buf = NULL;
	size_t keyword_len, rec_offset, name_offset, name_offset_root;
	unsigned int keywords_count;
	int ret;

	if (!map->write_to_disk) {
		/* if we crash in the middle of writing the header, the
		   keywords are more or less corrupted. avoid that by
		   making sure the header is updated atomically. */
		map = mail_index_map_clone(map, map->hdr.record_size);
		mail_index_sync_replace_map(ctx, map);
	}
	i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));

	ext_id = mail_index_map_lookup_ext(map, "keywords");
	if (ext_id != (uint32_t)-1) {
		/* update existing header */
		ext = array_idx(&map->extensions, ext_id);
{'fraction_non_alphanumeric': 0.12126582278481013, 'fraction_numerical': 0.010126582278481013, 'mean_word_length': 4.809558823529412, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3299976', 'n_tokens_mistral': 3201, 'n_tokens_neox': 2949, 'n_words': 651}