<commit_before>#include <stdlib.h>
#include <stdint.h>
#include <assert.h>
#include <stdbool.h>

#include "utf8.h"

/** 1 = space, 2 = punct, 3 = digit, 4 = alpha, 0 = other
 */
static const uint8_t ctype_class[256] = {
    /*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f */
    /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,
    /* 1 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* 2 */ 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
    /* 3 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2,
    /* 4 */ 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
    /* 5 */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 2,
    /* 6 */ 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
    /* 7 */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 0,
    /* 8 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* 9 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* a */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* b */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* c */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* d */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* e */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* f */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

bool rinku_isspace(char c) { return ctype_class[(uint8_t)c] == 1; }
bool rinku_ispunct(char c) { return ctype_class[(uint8_t)c] == 2; }
bool rinku_isdigit(char c) { return ctype_class[(uint8_t)c] == 3; }
bool rinku_isalpha(char c) { return ctype_class[(uint8_t)c] == 4; }
bool rinku_isalnum(char c)
{
	uint8_t cls = ctype_class[(uint8_t)c];
	return (cls == 3 || cls == 4);
}

static const int8_t utf8proc_utf8class[256] = {
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
    2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
    4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0};

int32_t utf8proc_next(const uint8_t *str, size_t *pos)
{
	int8_t length;

	str = &str[*pos];
	length = utf8proc_utf8class[str[0]];
	(*pos) += length;

	switch (length) {
	case 1:
		return str[0];
	case 2:
		return ((str[0] & 0x1F) << 6) + (str[1] & 0x3F);
	case 3:
		return ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) << 6) +
			(str[2] & 0x3F);
	case 4:
		return ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12) +
			((str[2] & 0x3F) << 6) + (str[3] & 0x3F);
	}

	return 0;
}

size_t utf8proc_find_space(const uint8_t *str, size_t pos, size_t size)
{
	while (pos < size) {
		const size_t last = pos;
		int32_t uc = utf8proc_next(str, &pos);
		if (utf8proc_is_space(uc))
			return last;
	}
	return size;
}

int32_t
utf8proc_rewind(const uint8_t *data, size_t pos)
{
	int32_t uc;

	if (!pos)
		return 0x0;

	if ((data[pos - 1] & 0xC0) == 0x0)
		return data[pos - 1];

	if (pos > 1 && utf8proc_utf8class[data[pos - 2]] == 2) {
		data = &data[pos - 2];
		uc = ((data[0] & 0x1F) << 6) + (data[1] & 0x3F);
		return (uc < 0x80) ? -1 : uc;
	}

	if (pos > 2 && utf8proc_utf8class[data[pos - 3]] == 3) {
		data = &data[pos - 3];
		uc = ((data[0] & 0x0F) << 12) + ((data[1] & 0x3F) << 6) +
			(data[2] & 0x3F);

		if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000))
			return -1;
		return uc;
	}

	if (pos > 3 && utf8proc_utf8class[data[pos - 4]] == 4) {
		data = &data[pos - 4];
		uc = ((data[0] & 0x07) << 18) + ((data[1] & 0x3F) << 12) +
			((data[2] & 0x3F) << 6) + (data[3] & 0x3F);

		if (uc < 0x10000 || uc >= 0x110000)
			return -1;
		return uc;
	}

	return -1;<commit_msg>Assume UTF8 strings are always well-formed<commit_after>#include <stdlib.h>
#include <stdint.h>
#include <assert.h>
#include <stdbool.h>

#include "utf8.h"

/** 1 = space, 2 = punct, 3 = digit, 4 = alpha, 0 = other
 */
static const uint8_t ctype_class[256] = {
    /*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f */
    /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,
    /* 1 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* 2 */ 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
    /* 3 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2,
    /* 4 */ 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
    /* 5 */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 2,
    /* 6 */ 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
    /* 7 */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 2, 2, 2, 2, 0,
    /* 8 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* 9 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* a */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* b */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* c */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* d */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* e */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    /* f */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

bool rinku_isspace(char c) { return ctype_class[(uint8_t)c] == 1; }
bool rinku_ispunct(char c) { return ctype_class[(uint8_t)c] == 2; }
bool rinku_isdigit(char c) { return ctype_class[(uint8_t)c] == 3; }
bool rinku_isalpha(char c) { return ctype_class[(uint8_t)c] == 4; }
bool rinku_isalnum(char c)
{
	uint8_t cls = ctype_class[(uint8_t)c];
	return (cls == 3 || cls == 4);
}

static const int8_t utf8proc_utf8class[256] = {
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
    1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
    2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
    4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0};

static int32_t read_cp(const uint8_t *str, int8_t length)
{
	switch (length) {
	case 1:
		return str[0];
	case 2:
		return ((str[0] & 0x1F) << 6) + (str[1] & 0x3F);
	case 3:
		return ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) << 6) +
			(str[2] & 0x3F);
	case 4:
		return ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12) +
			((str[2] & 0x3F) << 6) + (str[3] & 0x3F);
	default:
		return 0xFFFD; // replacement character
	}
}

int32_t utf8proc_next(const uint8_t *str, size_t *pos)
{
	const size_t p = *pos;
	const int8_t length = utf8proc_utf8class[str[p]];
	(*pos) += length;
	return read_cp(str + p, length);
}

size_t utf8proc_find_space(const uint8_t *str, size_t pos, size_t size)
{
	while (pos < size) {
		const size_t last = pos;
		int32_t uc = utf8proc_next(str, &pos);
		if (utf8proc_is_space(uc))
			return last;
	}
	return size;
}

int32_t utf8proc_rewind(const uint8_t *data, size_t pos)
{
	int8_t length = 0;

	if (!pos)
		return 0x0;

	if ((data[pos - 1] & 0xC0) == 0x0)
		return data[pos - 1];

	if (pos > 1 && utf8proc_utf8class[data[pos - 2]] == 2)
		length = 2;
	else if (pos > 2 && utf8proc_utf8class[data[pos - 3]] == 3)
		length = 3;
	else if (pos > 3 && utf8proc_utf8class[data[pos - 4]] == 4)
		length = 4;

	return read_cp(&data[pos - length], length);
{'fraction_non_alphanumeric': 0.24828025477707005, 'fraction_numerical': 0.17464968152866242, 'mean_word_length': 2.3042929292929295, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2591391', 'n_tokens_mistral': 5522, 'n_tokens_neox': 4258, 'n_words': 1546}