<commit_before>/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "str-sanitize.h"
#include "nfs-workarounds.h"
#include "mmap-util.h"
#include "read-full.h"
#include "mail-index-private.h"
#include "mail-index-sync-private.h"
#include "mail-index-modseq.h"
#include "mail-transaction-log-private.h"

static void mail_index_map_init_extbufs(struct mail_index_map *map,
					unsigned int initial_count)
{
#define EXTENSION_NAME_APPROX_LEN 20
#define EXT_GLOBAL_ALLOC_SIZE \
	((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
#define EXT_PER_ALLOC_SIZE \
	(EXTENSION_NAME_APPROX_LEN + \
	 sizeof(struct mail_index_ext) + sizeof(uint32_t))
	size_t size;

	if (map->extension_pool == NULL) {
		size = EXT_GLOBAL_ALLOC_SIZE +
			initial_count * EXT_PER_ALLOC_SIZE;
		map->extension_pool =
			pool_alloconly_create(MEMPOOL_GROWING"map extensions",
					      nearest_power(size));
	} else {
		p_clear(map->extension_pool);

		/* try to use the existing pool's size for initial_count so
		   we don't grow it unneededly */
		size = p_get_max_easy_alloc_size(map->extension_pool);
		if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
			initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
				EXT_PER_ALLOC_SIZE;
		}
	}

	p_array_init(&map->extensions, map->extension_pool, initial_count);
	p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
}

bool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
			       uint32_t *idx_r)
{
	const struct mail_index_ext *extensions;
	unsigned int i, size;

	if (array_is_created(&map->extensions)) {
		extensions = array_get(&map->extensions, &size);
		for (i = 0; i < size; i++) {
			if (strcmp(extensions[i].name, name) == 0) {
				if (idx_r != NULL)
					*idx_r = i;
				return TRUE;
			}
		}
	}
	return FALSE;
}

unsigned int mail_index_map_ext_hdr_offset(unsigned int name_len)
{
	size_t size = sizeof(struct mail_index_ext_header) + name_len;
	return MAIL_INDEX_HEADER_SIZE_ALIGN(size);
}

uint32_t
mail_index_map_register_ext(struct mail_index_map *map,
			    const char *name, uint32_t ext_offset,
			    const struct mail_index_ext_header *ext_hdr)
{
	struct mail_index_ext *ext;
	uint32_t idx, empty_idx = (uint32_t)-1;

	if (!array_is_created(&map->extensions)) {
                mail_index_map_init_extbufs(map, 5);
		idx = 0;
	} else {
		idx = array_count(&map->extensions);
	}
	i_assert(!mail_index_map_lookup_ext(map, name, NULL));

	ext = array_append_space(&map->extensions);
	ext->name = p_strdup(map->extension_pool, name);
	ext->ext_offset = ext_offset;
	ext->hdr_offset = ext_offset +
		mail_index_map_ext_hdr_offset(strlen(name));
	ext->hdr_size = ext_hdr->hdr_size;
	ext->record_offset = ext_hdr->record_offset;
	ext->record_size = ext_hdr->record_size;
	ext->record_align = ext_hdr->record_align;
	ext->reset_id = ext_hdr->reset_id;

	ext->index_idx = mail_index_ext_register(map->index, name,
						 ext_hdr->hdr_size,
						 ext_hdr->record_size,
						 ext_hdr->record_align);

	/* Update index ext_id -> map ext_id mapping. Fill non-used
	   ext_ids with (uint32_t)-1 */
	while (array_count(&map->ext_id_map) < ext->index_idx)
		array_append(&map->ext_id_map, &empty_idx, 1);
	array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
	return idx;
}

int mail_index_map_ext_get_next(struct mail_index_map *map,
				unsigned int *offset_p,
				const struct mail_index_ext_header **ext_hdr_r,
				const char **name_r)
{
	const struct mail_index_ext_header *ext_hdr;
	unsigned int offset, name_offset;

	offset = *offset_p;
	*name_r = "";

	/* Extension header contains:
	   - struct mail_index_ext_header
	   - name (not 0-terminated)
	   - 64bit alignment padding
	   - extension header contents
	   - 64bit alignment padding
	*/
	name_offset = offset + sizeof(*ext_hdr);
	ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
	if (offset + sizeof(*ext_hdr) >= map->hdr.header_size)
		return -1;

	offset += mail_index_map_ext_hdr_offset(ext_hdr->name_size);
	if (offset > map->hdr.header_size)
		return -1;

	*name_r = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
			    ext_hdr->name_size);
	if (strcmp(*name_r, str_sanitize(*name_r, -1)) != 0) {
		/* we allow only plain ASCII names, so this extension
		   is most likely broken */
		*name_r = "";
	}

	/* finally make sure that the hdr_size is small enough.
	   do this last so that we could return a usable name. */
	offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
	if (offset > map->hdr.header_size)
		return -1;

	*offset_p = offset;
	*ext_hdr_r = ext_hdr;
	return 0;
}

int mail_index_map_ext_hdr_check(const struct mail_index_header *hdr,
				 const struct mail_index_ext_header *ext_hdr,
				 const char *name, const char **error_r)
{
	if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
	    (ext_hdr->record_align == 0 && ext_hdr->record_size != 0)) {
		*error_r = "Invalid field values";
		return -1;
	}
	if (*name == '\0') {
		*error_r = "Broken name";
		return -1;
	}

	if (ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) {
		*error_r = t_strdup_printf("Record field points "
					   "outside record size (%u+%u > %u)",
					   ext_hdr->record_offset,
					   ext_hdr->record_size,
					   hdr->record_size);
		return -1;
	}

	if (ext_hdr->record_size > 0 &&
	    (ext_hdr->record_offset % ext_hdr->record_align) != 0) {
		*error_r = t_strdup_printf("Record field alignment %u "
					   "not used", ext_hdr->record_align);
		return -1;
	}
	/* if we get here from extension introduction, record_offset=0 and
	   hdr->record_size hasn't been updated yet */<commit_msg>mail_index_map_ext_hdr_check(): Fixed a wrong check.<commit_after>/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "array.h"
#include "str-sanitize.h"
#include "nfs-workarounds.h"
#include "mmap-util.h"
#include "read-full.h"
#include "mail-index-private.h"
#include "mail-index-sync-private.h"
#include "mail-index-modseq.h"
#include "mail-transaction-log-private.h"

static void mail_index_map_init_extbufs(struct mail_index_map *map,
					unsigned int initial_count)
{
#define EXTENSION_NAME_APPROX_LEN 20
#define EXT_GLOBAL_ALLOC_SIZE \
	((sizeof(map->extensions) + BUFFER_APPROX_SIZE) * 2)
#define EXT_PER_ALLOC_SIZE \
	(EXTENSION_NAME_APPROX_LEN + \
	 sizeof(struct mail_index_ext) + sizeof(uint32_t))
	size_t size;

	if (map->extension_pool == NULL) {
		size = EXT_GLOBAL_ALLOC_SIZE +
			initial_count * EXT_PER_ALLOC_SIZE;
		map->extension_pool =
			pool_alloconly_create(MEMPOOL_GROWING"map extensions",
					      nearest_power(size));
	} else {
		p_clear(map->extension_pool);

		/* try to use the existing pool's size for initial_count so
		   we don't grow it unneededly */
		size = p_get_max_easy_alloc_size(map->extension_pool);
		if (size > EXT_GLOBAL_ALLOC_SIZE + EXT_PER_ALLOC_SIZE) {
			initial_count = (size - EXT_GLOBAL_ALLOC_SIZE) /
				EXT_PER_ALLOC_SIZE;
		}
	}

	p_array_init(&map->extensions, map->extension_pool, initial_count);
	p_array_init(&map->ext_id_map, map->extension_pool, initial_count);
}

bool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
			       uint32_t *idx_r)
{
	const struct mail_index_ext *extensions;
	unsigned int i, size;

	if (array_is_created(&map->extensions)) {
		extensions = array_get(&map->extensions, &size);
		for (i = 0; i < size; i++) {
			if (strcmp(extensions[i].name, name) == 0) {
				if (idx_r != NULL)
					*idx_r = i;
				return TRUE;
			}
		}
	}
	return FALSE;
}

unsigned int mail_index_map_ext_hdr_offset(unsigned int name_len)
{
	size_t size = sizeof(struct mail_index_ext_header) + name_len;
	return MAIL_INDEX_HEADER_SIZE_ALIGN(size);
}

uint32_t
mail_index_map_register_ext(struct mail_index_map *map,
			    const char *name, uint32_t ext_offset,
			    const struct mail_index_ext_header *ext_hdr)
{
	struct mail_index_ext *ext;
	uint32_t idx, empty_idx = (uint32_t)-1;

	if (!array_is_created(&map->extensions)) {
                mail_index_map_init_extbufs(map, 5);
		idx = 0;
	} else {
		idx = array_count(&map->extensions);
	}
	i_assert(!mail_index_map_lookup_ext(map, name, NULL));

	ext = array_append_space(&map->extensions);
	ext->name = p_strdup(map->extension_pool, name);
	ext->ext_offset = ext_offset;
	ext->hdr_offset = ext_offset +
		mail_index_map_ext_hdr_offset(strlen(name));
	ext->hdr_size = ext_hdr->hdr_size;
	ext->record_offset = ext_hdr->record_offset;
	ext->record_size = ext_hdr->record_size;
	ext->record_align = ext_hdr->record_align;
	ext->reset_id = ext_hdr->reset_id;

	ext->index_idx = mail_index_ext_register(map->index, name,
						 ext_hdr->hdr_size,
						 ext_hdr->record_size,
						 ext_hdr->record_align);

	/* Update index ext_id -> map ext_id mapping. Fill non-used
	   ext_ids with (uint32_t)-1 */
	while (array_count(&map->ext_id_map) < ext->index_idx)
		array_append(&map->ext_id_map, &empty_idx, 1);
	array_idx_set(&map->ext_id_map, ext->index_idx, &idx);
	return idx;
}

int mail_index_map_ext_get_next(struct mail_index_map *map,
				unsigned int *offset_p,
				const struct mail_index_ext_header **ext_hdr_r,
				const char **name_r)
{
	const struct mail_index_ext_header *ext_hdr;
	unsigned int offset, name_offset;

	offset = *offset_p;
	*name_r = "";

	/* Extension header contains:
	   - struct mail_index_ext_header
	   - name (not 0-terminated)
	   - 64bit alignment padding
	   - extension header contents
	   - 64bit alignment padding
	*/
	name_offset = offset + sizeof(*ext_hdr);
	ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset);
	if (offset + sizeof(*ext_hdr) >= map->hdr.header_size)
		return -1;

	offset += mail_index_map_ext_hdr_offset(ext_hdr->name_size);
	if (offset > map->hdr.header_size)
		return -1;

	*name_r = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset),
			    ext_hdr->name_size);
	if (strcmp(*name_r, str_sanitize(*name_r, -1)) != 0) {
		/* we allow only plain ASCII names, so this extension
		   is most likely broken */
		*name_r = "";
	}

	/* finally make sure that the hdr_size is small enough.
	   do this last so that we could return a usable name. */
	offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size);
	if (offset > map->hdr.header_size)
		return -1;

	*offset_p = offset;
	*ext_hdr_r = ext_hdr;
	return 0;
}

int mail_index_map_ext_hdr_check(const struct mail_index_header *hdr,
				 const struct mail_index_ext_header *ext_hdr,
				 const char *name, const char **error_r)
{
	if ((ext_hdr->record_size == 0 && ext_hdr->hdr_size == 0) ||
	    (ext_hdr->record_align == 0 && ext_hdr->record_size != 0)) {
		*error_r = "Invalid field values";
		return -1;
	}
	if (*name == '\0') {
		*error_r = "Broken name";
		return -1;
	}

	/* if we get here from extension introduction, record_offset=0 and
	   hdr->record_size hasn't been updated yet */
	if (ext_hdr->record_offset != 0 &&
	    ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) {
		*error_r = t_strdup_printf("Record field points "
					   "outside record size (%u+%u > %u)",
					   ext_hdr->record_offset,
					   ext_hdr->record_size,
					   hdr->record_size);
		return -1;
	}

	if (ext_hdr->record_size > 0 &&
	    (ext_hdr->record_offset % ext_hdr->record_align) != 0) {
		*error_r = t_strdup_printf("Record field alignment %u "
					   "not used", ext_hdr->record_align);
		return -1;
	}
{'fraction_non_alphanumeric': 0.12485711773498637, 'fraction_numerical': 0.009759957794777103, 'mean_word_length': 4.2560073937153415, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 142, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4098102', 'n_tokens_mistral': 4595, 'n_tokens_neox': 4195, 'n_words': 876}