<commit_before>/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
/*  GMime
 *  Copyright (C) 2000-2007 <NAME>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */


#include <stdio.h>
#include <string.h>
#include <ctype.h>

enum {
	FALSE,
	TRUE
};

#define CHARS_LWSP " \t\n\r"               /* linear whitespace chars */
#define CHARS_TSPECIAL "()<>@,;:\\\"/[]?="
#define CHARS_SPECIAL "()<>@,;:\\\".[]"
#define CHARS_CSPECIAL "()\\\r"	           /* not in comments */
#define CHARS_DSPECIAL "[]\\\r \t"	   /* not in domains */
#define CHARS_ESPECIAL "()<>@,;:\"/[]?.=_" /* encoded word specials (rfc2047 5.1) */
#define CHARS_PSPECIAL "!*+-/=_"           /* encoded phrase specials (rfc2047 5.3) */
#define CHARS_ATTRCHAR "*'% "              /* attribute-char from rfc2184 */

static unsigned short gmime_special_table[256];

enum {
	IS_CTRL		= (1 << 0),
	IS_LWSP		= (1 << 1),
	IS_TSPECIAL	= (1 << 2),
	IS_SPECIAL	= (1 << 3),
	IS_SPACE	= (1 << 4),
	IS_DSPECIAL	= (1 << 5),
	IS_QPSAFE	= (1 << 6),
	IS_ESAFE	= (1 << 7),  /* encoded word safe */
	IS_PSAFE	= (1 << 8),  /* encoded word in phrase safe */
	IS_ATTRCHAR     = (1 << 9)   /* attribute-char from rfc2184 */
};

#define is_ctrl(x) ((gmime_special_table[(unsigned char)(x)] & IS_CTRL) != 0)
#define is_lwsp(x) ((gmime_special_table[(unsigned char)(x)] & IS_LWSP) != 0)
#define is_tspecial(x) ((gmime_special_table[(unsigned char)(x)] & IS_TSPECIAL) != 0)
#define is_type(x, t) ((gmime_special_table[(unsigned char)(x)] & (t)) != 0)
#define is_ttoken(x) ((gmime_special_table[(unsigned char)(x)] & (IS_TSPECIAL|IS_LWSP|IS_CTRL)) == 0)
#define is_atom(x) ((gmime_special_table[(unsigned char)(x)] & (IS_SPECIAL|IS_SPACE|IS_CTRL)) == 0)
#define is_dtext(x) ((gmime_special_table[(unsigned char)(x)] & IS_DSPECIAL) == 0)
#define is_fieldname(x) ((gmime_special_table[(unsigned char)(x)] & (IS_CTRL|IS_SPACE)) == 0)
#define is_qpsafe(x) ((gmime_special_table[(unsigned char)(x)] & IS_QPSAFE) != 0)
#define is_especial(x) ((gmime_special_table[(unsigned char)(x)] & IS_ESAFE) != 0)
#define is_psafe(x) ((gmime_special_table[(unsigned char)(x)] & IS_PSAFE) != 0)
#define is_attrchar(x) ((gmime_special_table[(unsigned char)(x)] & IS_ATTRCHAR) != 0)

/* code to rebuild the gmime_special_table */
static void
header_remove_bits (unsigned short bit, unsigned char *vals)
{
	int i;
	
	for (i = 0; vals[i]; i++)
		gmime_special_table[vals[i]] &= ~bit;
}

static void
header_init_bits (unsigned short bit, unsigned short bitcopy, int remove, unsigned char *vals)
{
	int i, len = strlen (vals);
	
	if (!remove) {
		for (i = 0; i < len; i++)
			gmime_special_table[vals[i]] |= bit;
		if (bitcopy) {
			for (i = 0; i < 256; i++) {
				if (gmime_special_table[i] & bitcopy)
					gmime_special_table[i] |= bit;
			}
		}
	} else {
		for (i = 0; i < 256; i++)
			gmime_special_table[i] |= bit;
		for (i = 0; i < len; i++)
			gmime_special_table[vals[i]] &= ~bit;
		if (bitcopy) {
			for (i = 0; i < 256; i++) {
				if (gmime_special_table[i] & bitcopy)
					gmime_special_table[i] &= ~bit;
			}
		}
	}
}

static void
header_decode_init (void)
{
	int i;
	
	for (i = 0; i < 256; i++) {
		gmime_special_table[i] = 0;
		if (i < 32 || i == 127)
			gmime_special_table[i] |= IS_CTRL;
		if (i > 32 && i < 127)<commit_msg>fixed signedness warnings<commit_after>/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
/*  GMime
 *  Copyright (C) 2000-2007 <NAME>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */


#include <stdio.h>
#include <string.h>
#include <ctype.h>

enum {
	FALSE,
	TRUE
};

#define CHARS_LWSP " \t\n\r"               /* linear whitespace chars */
#define CHARS_TSPECIAL "()<>@,;:\\\"/[]?="
#define CHARS_SPECIAL "()<>@,;:\\\".[]"
#define CHARS_CSPECIAL "()\\\r"	           /* not in comments */
#define CHARS_DSPECIAL "[]\\\r \t"	   /* not in domains */
#define CHARS_ESPECIAL "()<>@,;:\"/[]?.=_" /* encoded word specials (rfc2047 5.1) */
#define CHARS_PSPECIAL "!*+-/=_"           /* encoded phrase specials (rfc2047 5.3) */
#define CHARS_ATTRCHAR "*'% "              /* attribute-char from rfc2184 */

static unsigned short gmime_special_table[256];

enum {
	IS_CTRL		= (1 << 0),
	IS_LWSP		= (1 << 1),
	IS_TSPECIAL	= (1 << 2),
	IS_SPECIAL	= (1 << 3),
	IS_SPACE	= (1 << 4),
	IS_DSPECIAL	= (1 << 5),
	IS_QPSAFE	= (1 << 6),
	IS_ESAFE	= (1 << 7),  /* encoded word safe */
	IS_PSAFE	= (1 << 8),  /* encoded word in phrase safe */
	IS_ATTRCHAR     = (1 << 9)   /* attribute-char from rfc2184 */
};

#define is_ctrl(x) ((gmime_special_table[(unsigned char)(x)] & IS_CTRL) != 0)
#define is_lwsp(x) ((gmime_special_table[(unsigned char)(x)] & IS_LWSP) != 0)
#define is_tspecial(x) ((gmime_special_table[(unsigned char)(x)] & IS_TSPECIAL) != 0)
#define is_type(x, t) ((gmime_special_table[(unsigned char)(x)] & (t)) != 0)
#define is_ttoken(x) ((gmime_special_table[(unsigned char)(x)] & (IS_TSPECIAL|IS_LWSP|IS_CTRL)) == 0)
#define is_atom(x) ((gmime_special_table[(unsigned char)(x)] & (IS_SPECIAL|IS_SPACE|IS_CTRL)) == 0)
#define is_dtext(x) ((gmime_special_table[(unsigned char)(x)] & IS_DSPECIAL) == 0)
#define is_fieldname(x) ((gmime_special_table[(unsigned char)(x)] & (IS_CTRL|IS_SPACE)) == 0)
#define is_qpsafe(x) ((gmime_special_table[(unsigned char)(x)] & IS_QPSAFE) != 0)
#define is_especial(x) ((gmime_special_table[(unsigned char)(x)] & IS_ESAFE) != 0)
#define is_psafe(x) ((gmime_special_table[(unsigned char)(x)] & IS_PSAFE) != 0)
#define is_attrchar(x) ((gmime_special_table[(unsigned char)(x)] & IS_ATTRCHAR) != 0)

/* code to rebuild the gmime_special_table */
static void
header_remove_bits (unsigned short bit, char *vals)
{
	register unsigned char *inptr = (unsigned char *) vals;
	
	while (*inptr != '\0')
		gmime_special_table[*inptr++] &= ~bit;
}

static void
header_init_bits (unsigned short bit, unsigned short bitcopy, int remove, char *vals)
{
	register unsigned char *inptr = (unsigned char *) vals;
	int i;
	
	if (!remove) {
		while (*inptr != '\0')
			gmime_special_table[*inptr++] |= bit;
		if (bitcopy) {
			for (i = 0; i < 256; i++) {
				if (gmime_special_table[i] & bitcopy)
					gmime_special_table[i] |= bit;
			}
		}
	} else {
		for (i = 0; i < 256; i++)
			gmime_special_table[i] |= bit;
		while (*inptr != '\0')
			gmime_special_table[*inptr++] &= ~bit;
		if (bitcopy) {
			for (i = 0; i < 256; i++) {
				if (gmime_special_table[i] & bitcopy)
					gmime_special_table[i] &= ~bit;
			}
		}
	}
}

static void
header_decode_init (void)
{
	int i;
	
	for (i = 0; i < 256; i++) {
		gmime_special_table[i] = 0;
		if (i < 32 || i == 127)
			gmime_special_table[i] |= IS_CTRL;
		if (i > 32 && i < 127)
{'fraction_non_alphanumeric': 0.16923462357509708, 'fraction_numerical': 0.026806964800200426, 'mean_word_length': 3.8476017000607166, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6492701', 'n_tokens_mistral': 3351, 'n_tokens_neox': 3007, 'n_words': 834}