<commit_before>#include <utf8proc.h>

#include "tm.h"

size_t tm_ucs2_str_length (const uint8_t* buf, ssize_t buf_len)
{
	// This is silly, don't do this.
	if (buf_len < 0) {
		buf_len = strlen((char*) buf);
	}

	const uint8_t* ptr = buf;
	uint8_t truelen = 0;
	int32_t dst = 0;
	for (;;) {
		ssize_t bytes_read = utf8proc_iterate(ptr, buf_len, &dst);
		if (dst == -1) {
			break;
		}
		ptr += bytes_read;
		buf_len -= bytes_read;
		truelen += dst & 0x10000 ? 2 : 1;
	}
	return truelen;
}

int32_t tm_ucs2_str_charat (const uint8_t* buf, ssize_t buf_len, ssize_t index)
{
	// This is silly, don't do this.
	if (buf_len < 0) {
		buf_len = strlen((char*) buf);
	}

	const uint8_t* ptr = buf;
	int32_t dst = 0;
	while (index >= 0) {
		ssize_t bytes_read = utf8proc_iterate(ptr, buf_len, &dst);
		if (dst == -1) {
			return dst;
		}
		ptr += bytes_read;
		buf_len -= bytes_read;
		index -= dst & 0x10000 ? 2 : 1;
	}

	return index == -2
		? (dst - 0x10000) / 0x400 + 0xD800
		: dst > 0xFFFF
			? (dst - 0x10000) % 0x400 + 0xDC00
			: dst & 0xFFFF;
}

// convert UCS-2 index to offset in CESU-8 string
size_t tm_ucs2_str_lookup_16to8 (const uint8_t* buf, size_t len, size_t ucs2_index, size_t* seq_len)
{
  const uint8_t* const orig_buf = buf;
  ssize_t bytes_read;
  size_t ucs2_position = 0;
	while (ucs2_position <= ucs2_index) {        // NOTE: we read _past_ ucs2_index to get seq_len
    if (len == 0) {
      bytes_read = 0;
      break;
    }
    int32_t uchar;
		bytes_read = utf8proc_iterate(buf, len, &uchar);
    if (uchar < 0) {
      // "convert" ptr[0] to noncharacter
      uchar = 0xFFFF;
      bytes_read = 1;
    }
		buf += bytes_read;
		len -= bytes_read;
    // NOTE: this logic (*somewhat* incorrectly) won't split surrogate pair if `buf` is UTF-8 instead of CESU-8
    //       we can't do any better here; IMO the correct solution is for colony-compiler to ensure CESU for us
    ucs2_position += (uchar > 0xFFFF) ? 2 : 1;
	}
  *seq_len = bytes_read;
	return (buf - bytes_read) - orig_buf;
}


// convert offset in UTF-8/CESU-8 string to UCS-2 index [converts lengths, really]<commit_msg>fix replacement vs. noncharacter, add note about overall futility of that effort anyway<commit_after>#include <utf8proc.h>

#include "tm.h"

size_t tm_ucs2_str_length (const uint8_t* buf, ssize_t buf_len)
{
	// This is silly, don't do this.
	if (buf_len < 0) {
		buf_len = strlen((char*) buf);
	}

	const uint8_t* ptr = buf;
	uint8_t truelen = 0;
	int32_t dst = 0;
	for (;;) {
		ssize_t bytes_read = utf8proc_iterate(ptr, buf_len, &dst);
		if (dst == -1) {
			break;
		}
		ptr += bytes_read;
		buf_len -= bytes_read;
		truelen += dst & 0x10000 ? 2 : 1;
	}
	return truelen;
}

int32_t tm_ucs2_str_charat (const uint8_t* buf, ssize_t buf_len, ssize_t index)
{
	// This is silly, don't do this.
	if (buf_len < 0) {
		buf_len = strlen((char*) buf);
	}

	const uint8_t* ptr = buf;
	int32_t dst = 0;
	while (index >= 0) {
		ssize_t bytes_read = utf8proc_iterate(ptr, buf_len, &dst);
		if (dst == -1) {
			return dst;
		}
		ptr += bytes_read;
		buf_len -= bytes_read;
		index -= dst & 0x10000 ? 2 : 1;
	}

	return index == -2
		? (dst - 0x10000) / 0x400 + 0xD800
		: dst > 0xFFFF
			? (dst - 0x10000) % 0x400 + 0xDC00
			: dst & 0xFFFF;
}

// convert UCS-2 index to offset in CESU-8 string
size_t tm_ucs2_str_lookup_16to8 (const uint8_t* buf, size_t len, size_t ucs2_index, size_t* seq_len)
{
  const uint8_t* const orig_buf = buf;
  ssize_t bytes_read;
  size_t ucs2_position = 0;
	while (ucs2_position <= ucs2_index) {        // NOTE: we read _past_ ucs2_index to get seq_len
    if (len == 0) {
      bytes_read = 0;
      break;
    }
    int32_t uchar;
		bytes_read = utf8proc_iterate(buf, len, &uchar);
    if (uchar < 0) {      // NOTE: it seems colony-compiler is sanitizing, so (currently) this shouldn't happen
      // "replace" ptr[0]
      uchar = 0xFFFD;
      bytes_read = 1;
    }
		buf += bytes_read;
		len -= bytes_read;
    // NOTE: this logic (*somewhat* incorrectly) won't split surrogate pair if `buf` is UTF-8 instead of CESU-8
    //       we can't do any better here; IMO the correct solution is for colony-compiler to ensure CESU for us
    ucs2_position += (uchar > 0xFFFF) ? 2 : 1;
	}
  *seq_len = bytes_read;
	return (buf - bytes_read) - orig_buf;
}


// convert offset in UTF-8/CESU-8 string to UCS-2 index [converts lengths, really]
{'fraction_non_alphanumeric': 0.1237680495072198, 'fraction_numerical': 0.04675681870272748, 'mean_word_length': 2.9672727272727273, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7506051', 'n_tokens_mistral': 1981, 'n_tokens_neox': 1777, 'n_words': 504}