<commit_before>/* Copyright (C) 2002 <NAME> */

/* Implementated against draft-ietf-imapext-sort-10 and
   draft-ietf-imapext-thread-12 */

#include "lib.h"
#include "buffer.h"
#include "charset-utf8.h"
#include "message-header-decode.h"
#include "imap-base-subject.h"

static int header_decode(const unsigned char *data, size_t size,
			 const char *charset, void *context)
{
	buffer_t *buf = context;
	const char *utf8_str;
	unsigned char *buf_data;
	size_t i, used_size;

	if (charset == NULL) {
		/* It's ASCII. */
		buffer_append(buf, data, size);
	} else {
		t_push();
		utf8_str = charset_to_utf8_string(charset, NULL, data,
						  size, &size);
		if (utf8_str == NULL)
			size = 0;
		else
			buffer_append(buf, utf8_str, size);
		t_pop();
	}

	if (size > 0) {
		/* @UNSAFE: uppercase it. Current draft specifies that we
		   should touch only ASCII. */
		buf_data = buffer_get_modifyable_data(buf, &used_size);
		for (i = used_size - size; i < used_size; i++) {
			if (buf_data[i] >= 'a' && buf_data[i] <= 'z')
				buf_data[i] = buf_data[i] - 'a' + 'A';
		}
	}

	return TRUE;
}

static void pack_whitespace(buffer_t *buf)
{
	char *data, *dest;
	int last_lwsp;

	data = buffer_get_modifyable_data(buf, NULL);

	/* check if we need to do anything */
	while (*data != '\0') {
		if (*data == '\t' ||
		    (*data == ' ' && (data[1] == ' ' || data[1] == '\t')))
			break;
		data++;
	}

	if (*data == '\0')
		return;

	/* @UNSAFE: convert/pack the whitespace */
	dest = data; last_lwsp = FALSE;
	while (*data != '\0') {
		if (*data == '\t' || *data == ' ') {
			if (!last_lwsp) {
				*dest++ = ' ';
				last_lwsp = TRUE;
			}
		} else {
			*dest++ = *data;
			last_lwsp = FALSE;
		}
		data++;
	}
	*dest = '\0';

	data = buffer_get_modifyable_data(buf, NULL);
	buffer_set_used_size(buf, (size_t) (dest - data)+1);
}

static void remove_subj_trailers(buffer_t *buf, int *is_reply_or_forward)
{
	const char *data;
	size_t orig_size, size;

	/* subj-trailer    = "(fwd)" / WSP */
	data = buffer_get_data(buf, &orig_size);

	if (orig_size < 2) /* size includes trailing \0 */
		return;
<commit_msg>Strip CR and LF characters too.<commit_after>/* Copyright (C) 2002 <NAME> */

/* Implementated against draft-ietf-imapext-sort-10 and
   draft-ietf-imapext-thread-12 */

#include "lib.h"
#include "buffer.h"
#include "charset-utf8.h"
#include "message-header-decode.h"
#include "imap-base-subject.h"

static int header_decode(const unsigned char *data, size_t size,
			 const char *charset, void *context)
{
	buffer_t *buf = context;
	const char *utf8_str;
	unsigned char *buf_data;
	size_t i, used_size;

	if (charset == NULL) {
		/* It's ASCII. */
		buffer_append(buf, data, size);
	} else {
		t_push();
		utf8_str = charset_to_utf8_string(charset, NULL, data,
						  size, &size);
		if (utf8_str == NULL)
			size = 0;
		else
			buffer_append(buf, utf8_str, size);
		t_pop();
	}

	if (size > 0) {
		/* @UNSAFE: uppercase it. Current draft specifies that we
		   should touch only ASCII. */
		buf_data = buffer_get_modifyable_data(buf, &used_size);
		for (i = used_size - size; i < used_size; i++) {
			if (buf_data[i] >= 'a' && buf_data[i] <= 'z')
				buf_data[i] = buf_data[i] - 'a' + 'A';
		}
	}

	return TRUE;
}

static void pack_whitespace(buffer_t *buf)
{
	char *data, *dest;
	int last_lwsp;

	data = buffer_get_modifyable_data(buf, NULL);

	/* check if we need to do anything */
	while (*data != '\0') {
		if (*data == '\t' || *data == '\n' || *data == '\r' ||
		    (*data == ' ' && (data[1] == ' ' || data[1] == '\t')))
			break;
		data++;
	}

	if (*data == '\0')
		return;

	/* @UNSAFE: convert/pack the whitespace */
	dest = data; last_lwsp = FALSE;
	while (*data != '\0') {
		if (*data == '\t' || *data == ' ' ||
		    *data == '\r' || *data == '\n') {
			if (!last_lwsp) {
				*dest++ = ' ';
				last_lwsp = TRUE;
			}
		} else {
			*dest++ = *data;
			last_lwsp = FALSE;
		}
		data++;
	}
	*dest = '\0';

	data = buffer_get_modifyable_data(buf, NULL);
	buffer_set_used_size(buf, (size_t) (dest - data)+1);
}

static void remove_subj_trailers(buffer_t *buf, int *is_reply_or_forward)
{
	const char *data;
	size_t orig_size, size;

	/* subj-trailer    = "(fwd)" / WSP */
	data = buffer_get_data(buf, &orig_size);

	if (orig_size < 2) /* size includes trailing \0 */
		return;

{'fraction_non_alphanumeric': 0.18090569561157796, 'fraction_numerical': 0.011671335200746966, 'mean_word_length': 3.230009871668312, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5503959', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1722, 'n_words': 419}