<commit_before>/* Copyright (C) 2002 <NAME> */

#include "lib.h"
#include "base64.h"
#include "buffer.h"
#include "charset-utf8.h"
#include "quoted-printable.h"
#include "message-parser.h"
#include "message-header-decode.h"
#include "message-header-search.h"

#include <ctype.h>

struct header_search_context {
	pool_t pool;

	unsigned char *key;
	size_t key_len;
	char *key_charset;

	buffer_t *match_buf;

	unsigned int found:1;
	unsigned int last_newline:1;
	unsigned int submatch:1;
	unsigned int key_ascii:1;
	unsigned int unknown_charset:1;
};

static void search_loop(const unsigned char *data, size_t size,
			struct header_search_context *ctx);

struct header_search_context *
message_header_search_init(pool_t pool, const char *key, const char *charset,
			   int *unknown_charset)
{
	struct header_search_context *ctx;
	size_t key_len;
	const unsigned char *p;

	ctx = p_new(pool, struct header_search_context, 1);
	ctx->pool = pool;

	/* get the key uppercased */
	key = charset_to_ucase_utf8_string(charset, unknown_charset,
					   (const unsigned char *) key,
					   strlen(key), &key_len);

	if (key == NULL) {
		/* invalid key */
		p_free(pool, ctx);
		return NULL;
	}

	ctx->key = (unsigned char *) p_strdup(pool, key);
	ctx->key_len = key_len;
	ctx->key_charset = p_strdup(pool, charset);
	ctx->unknown_charset = charset == NULL;

	ctx->key_ascii = TRUE;
	for (p = ctx->key; *p != '\0'; p++) {
		if ((*p & 0x80) != 0) {
			ctx->key_ascii = FALSE;
			break;
		}
	}

	i_assert(ctx->key_len <= SSIZE_T_MAX/sizeof(size_t));
	ctx->match_buf = buffer_create_static_hard(pool, sizeof(size_t) *
						   ctx->key_len);
	return ctx;
}

void message_header_search_free(struct header_search_context *ctx)
{
	pool_t pool;

	buffer_free(ctx->match_buf);

	pool = ctx->pool;
	p_free(pool, ctx->key);
	p_free(pool, ctx->key_charset);
	p_free(pool, ctx);
}

static void search_with_charset(const unsigned char *data, size_t size,
				const char *charset,
				struct header_search_context *ctx)
{
	const char *utf8_data;
	size_t utf8_size;

	if (ctx->unknown_charset) {
		/* we don't know the source charset, so assume we want to
		   match using same charsets */
		charset = NULL;
	} else if (charset != NULL && strcasecmp(charset, "x-unknown") == 0) {
		/* compare with same charset as search key. the key is already
		   in utf-8 so we can't use charset = NULL comparing. */
		charset = ctx->key_charset;
	}

	utf8_data = charset_to_ucase_utf8_string(charset, NULL,
						 data, size, &utf8_size);

	if (utf8_data == NULL) {
		/* unknown character set, or invalid data */
	} else {
		ctx->submatch = TRUE;
		search_loop((const unsigned char *) utf8_data, utf8_size, ctx);
		ctx->submatch = FALSE;
	}
}<commit_msg>Try to compare strings even if charset conversion failed.<commit_after>/* Copyright (C) 2002 <NAME> */

#include "lib.h"
#include "base64.h"
#include "buffer.h"
#include "charset-utf8.h"
#include "quoted-printable.h"
#include "message-parser.h"
#include "message-header-decode.h"
#include "message-header-search.h"

#include <ctype.h>

struct header_search_context {
	pool_t pool;

	unsigned char *key;
	size_t key_len;
	char *key_charset;

	buffer_t *match_buf;

	unsigned int found:1;
	unsigned int last_newline:1;
	unsigned int submatch:1;
	unsigned int key_ascii:1;
	unsigned int unknown_charset:1;
};

static void search_loop(const unsigned char *data, size_t size,
			struct header_search_context *ctx);

struct header_search_context *
message_header_search_init(pool_t pool, const char *key, const char *charset,
			   int *unknown_charset)
{
	struct header_search_context *ctx;
	size_t key_len;
	const unsigned char *p;

	ctx = p_new(pool, struct header_search_context, 1);
	ctx->pool = pool;

	/* get the key uppercased */
	key = charset_to_ucase_utf8_string(charset, unknown_charset,
					   (const unsigned char *) key,
					   strlen(key), &key_len);

	if (key == NULL) {
		/* invalid key */
		p_free(pool, ctx);
		return NULL;
	}

	ctx->key = (unsigned char *) p_strdup(pool, key);
	ctx->key_len = key_len;
	ctx->key_charset = p_strdup(pool, charset);
	ctx->unknown_charset = charset == NULL;

	ctx->key_ascii = TRUE;
	for (p = ctx->key; *p != '\0'; p++) {
		if ((*p & 0x80) != 0) {
			ctx->key_ascii = FALSE;
			break;
		}
	}

	i_assert(ctx->key_len <= SSIZE_T_MAX/sizeof(size_t));
	ctx->match_buf = buffer_create_static_hard(pool, sizeof(size_t) *
						   ctx->key_len);
	return ctx;
}

void message_header_search_free(struct header_search_context *ctx)
{
	pool_t pool;

	buffer_free(ctx->match_buf);

	pool = ctx->pool;
	p_free(pool, ctx->key);
	p_free(pool, ctx->key_charset);
	p_free(pool, ctx);
}

static void search_with_charset(const unsigned char *data, size_t size,
				const char *charset,
				struct header_search_context *ctx)
{
	const char *utf8_data;
	size_t utf8_size;

	if (ctx->unknown_charset) {
		/* we don't know the source charset, so assume we want to
		   match using same charsets */
		charset = NULL;
	} else if (charset != NULL && strcasecmp(charset, "x-unknown") == 0) {
		/* compare with same charset as search key. the key is already
		   in utf-8 so we can't use charset = NULL comparing. */
		charset = ctx->key_charset;
	}

	utf8_data = charset_to_ucase_utf8_string(charset, NULL,
						 data, size, &utf8_size);

	if (utf8_data == NULL) {
		/* unknown character set, or invalid data. just compare it
		   directly so at least ASCII comparision works. */
		utf8_data = data;
		utf8_size = size;
	}

	ctx->submatch = TRUE;
	search_loop((const unsigned char *) utf8_data, utf8_size, ctx);
	ctx->submatch = FALSE;
}
{'fraction_non_alphanumeric': 0.1271064897812836, 'fraction_numerical': 0.010756543564001434, 'mean_word_length': 4.11365719523373, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7572280', 'n_tokens_mistral': 2218, 'n_tokens_neox': 1994, 'n_words': 533}