<commit_before>#include "gmqcc.h"

static unsigned char utf8_lengths[256] = {
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  * they could be interpreted as 2-byte starts but
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  * the codepoint would be < 127
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  *
	0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  * C0 and C1 would also result in overlong encodings
	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  */
	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
	4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	/* with F5 the codepoint is above 0x10FFFF,
	 * F8-FB would start 5-byte sequences
	 * FC-FD would start 6-byte sequences
	 * ...
	 */
};

static Uchar utf8_range[5] = {
	1,       /* invalid - let's not allow the creation of 0-bytes :P
	1,        * ascii minimum
	0x80,     * 2-byte minimum
	0x800,    * 3-byte minimum
	0x10000,  * 4-byte minimum */
};

/** Analyze the next character and return various information if requested.
 * @param _s      An utf-8 string.
 * @param _start  Filled with the start byte-offset of the next valid character
 * @param _len    Fileed with the length of the next valid character
 * @param _ch     Filled with the unicode value of the next character
 * @param _maxlen Maximum number of bytes to read from _s
 * @return        Whether or not another valid character is in the string
 */
static bool u8_analyze(const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen)
{
	const unsigned char *s = (const unsigned char*)_s;
	size_t i, j;
	size_t bits = 0;
	Uchar ch;

	i = 0;
findchar:
	while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0)
		++i;

	if (i >= _maxlen || !s[i]) {
		if (_start) *_start = i;
		if (_len) *_len = 0;
		return false;
	}

	if (bits == 1) { /* ascii */
		if (_start) *_start = i;
		if (_len) *_len = 1;
		if (_ch) *_ch = (Uchar)s[i];
		return true;
	}

	ch = (s[i] & (0xFF >> bits));
	for (j = 1; j < bits; ++j)
	{
		if ( (s[i+j] & 0xC0) != 0x80 )
		{
			i += j;
			goto findchar;
		}
		ch = (ch << 6) | (s[i+j] & 0x3F);
	}
	if (ch < utf8_range[bits] || ch >= 0x10FFFF)
	{
		i += bits;
		goto findchar;
	}

	if (_start)
		*_start = i;<commit_msg>u8_analyze: return false on any invalid character, we do not allow invalid/overlong characters in code at all<commit_after>#include "gmqcc.h"

static unsigned char utf8_lengths[256] = {
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  * they could be interpreted as 2-byte starts but
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  * the codepoint would be < 127
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  *
	0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  * C0 and C1 would also result in overlong encodings
	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  */
	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
	4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	/* with F5 the codepoint is above 0x10FFFF,
	 * F8-FB would start 5-byte sequences
	 * FC-FD would start 6-byte sequences
	 * ...
	 */
};

static Uchar utf8_range[5] = {
	1,       /* invalid - let's not allow the creation of 0-bytes :P
	1,        * ascii minimum
	0x80,     * 2-byte minimum
	0x800,    * 3-byte minimum
	0x10000,  * 4-byte minimum */
};

/** Analyze the next character and return various information if requested.
 * @param _s      An utf-8 string.
 * @param _start  Filled with the start byte-offset of the next valid character
 * @param _len    Fileed with the length of the next valid character
 * @param _ch     Filled with the unicode value of the next character
 * @param _maxlen Maximum number of bytes to read from _s
 * @return        Whether or not another valid character is in the string
 */
static bool u8_analyze(const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen)
{
	const unsigned char *s = (const unsigned char*)_s;
	size_t i, j;
	size_t bits = 0;
	Uchar ch;

	i = 0;
/* findchar: */
	while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0)
		++i;

	if (i >= _maxlen || !s[i]) {
		if (_start) *_start = i;
		if (_len) *_len = 0;
		return false;
	}

	if (bits == 1) { /* ascii */
		if (_start) *_start = i;
		if (_len) *_len = 1;
		if (_ch) *_ch = (Uchar)s[i];
		return true;
	}

	ch = (s[i] & (0xFF >> bits));
	for (j = 1; j < bits; ++j)
	{
		if ( (s[i+j] & 0xC0) != 0x80 )
		{
			i += j;
			/* in gmqcc, invalid / overlong encodings are considered an error
			 * goto findchar;
			 */
			return false;
		}
		ch = (ch << 6) | (s[i+j] & 0x3F);
	}
	if (ch < utf8_range[bits] || ch >= 0x10FFFF)
	{
		/* same: error
		 * i += bits;
		 * goto findchar;
		 */
		return false;
	}

	if (_start)
		*_start = i;
{'fraction_non_alphanumeric': 0.18576235169116345, 'fraction_numerical': 0.11386576943509828, 'mean_word_length': 2.503722084367246, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5639876', 'n_tokens_mistral': 3251, 'n_tokens_neox': 2597, 'n_words': 1052}