<commit_before>#include "common.h"
#undef UChar

#include "unicode/ucnv.h"
#include "unicode/utrans.h"

#ifndef UC_DECOMPOSITION_MAX_LENGTH
/* Maximum size of decomposition of a single Unicode character.  */
#define UC_DECOMPOSITION_MAX_LENGTH 32
#endif

#define DECOMPOSITION_SIZE(sz) (sizeof(UChar)*sz)*UC_DECOMPOSITION_MAX_LENGTH

extern VALUE rb_mCharlockHolmes;
static VALUE rb_cConverter;

static VALUE rb_converter_convert(VALUE self, VALUE rb_txt, VALUE rb_src_enc, VALUE rb_dst_enc) {
	VALUE rb_out;
	const char *src_enc;
	const char *dst_enc;
	const char *src_txt;
	char *out_buf;
	void *rb_enc = NULL;
	int32_t src_len;
	int32_t out_len;
	UErrorCode status = U_ZERO_ERROR;

	Check_Type(rb_txt, T_STRING);
	Check_Type(rb_src_enc, T_STRING);
	Check_Type(rb_dst_enc, T_STRING);

	src_txt = RSTRING_PTR(rb_txt);
	src_len = RSTRING_LEN(rb_txt);
	src_enc = RSTRING_PTR(rb_src_enc);
	dst_enc = RSTRING_PTR(rb_dst_enc);

	// first determin the size of the output buffer
	out_len = ucnv_convert(dst_enc, src_enc, NULL, 0, src_txt, src_len, &status);
	if (status != U_BUFFER_OVERFLOW_ERROR) {
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}
	out_buf = xmalloc(out_len);

	// now do the actual conversion
	status = U_ZERO_ERROR;
	out_len = ucnv_convert(dst_enc, src_enc, out_buf, out_len, src_txt, src_len, &status);
	if (!U_SUCCESS(status)) {
		xfree(out_buf);
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}

#ifdef HAVE_RUBY_ENCODING_H
	rb_enc = (void *)rb_enc_find(dst_enc);
#endif

	rb_out = charlock_new_enc_str(out_buf, out_len, rb_enc);

	xfree(out_buf);

	return rb_out;
}

// The return value needs to be freed later
static inline UChar *cstr_to_uchar(const char *src, int32_t src_len, int32_t *out_len) {
	UErrorCode status;
	UChar *dst_uchars;
	UConverter *utf8_conv;
	int32_t dst_uchar_len;

	status = U_ZERO_ERROR;
	// grab a UTF-8 converter
	utf8_conv = ucnv_open("UTF-8", &status);
	if (!U_SUCCESS(status)) {
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}

	status = U_ZERO_ERROR;
	// get the size of the buffer needed for the char -> UChar conversion
	dst_uchar_len = ucnv_toUChars(utf8_conv, NULL, 0, src, src_len, &status);
	if (status != U_BUFFER_OVERFLOW_ERROR) {
		ucnv_close(utf8_conv);
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}
	*out_len = dst_uchar_len;
	dst_uchars = xmalloc(DECOMPOSITION_SIZE(dst_uchar_len));

	status = U_ZERO_ERROR;
	// actually do the char -> UChar conversion
	dst_uchar_len = ucnv_toUChars(utf8_conv, dst_uchars, dst_uchar_len, src, src_len, &status);
	if (!U_SUCCESS(status)) {
		ucnv_close(utf8_conv);
		xfree(dst_uchars);
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}

	ucnv_close(utf8_conv);
	return dst_uchars;
}

// The return value needs to be freed later
static inline char *uchar_to_cstr(UChar *src, int32_t src_len, int32_t *out_len) {
	UErrorCode status;
	UConverter *utf8_conv;
	char *dst_cstr;
	int32_t dst_cstr_len;

	status = U_ZERO_ERROR;
	// grab a UTF-8 converter
	utf8_conv = ucnv_open("UTF-8", &status);
	if (!U_SUCCESS(status)) {
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}
<commit_msg>get rid of DECOMPOSITION_SIZE macro<commit_after>#include "common.h"
#undef UChar

#include "unicode/ucnv.h"
#include "unicode/utrans.h"

#ifndef UC_DECOMPOSITION_MAX_LENGTH
/* Maximum size of decomposition of a single Unicode character.  */
#define UC_DECOMPOSITION_MAX_LENGTH 32
#endif

extern VALUE rb_mCharlockHolmes;
static VALUE rb_cConverter;

static VALUE rb_converter_convert(VALUE self, VALUE rb_txt, VALUE rb_src_enc, VALUE rb_dst_enc) {
	VALUE rb_out;
	const char *src_enc;
	const char *dst_enc;
	const char *src_txt;
	char *out_buf;
	void *rb_enc = NULL;
	int32_t src_len;
	int32_t out_len;
	UErrorCode status = U_ZERO_ERROR;

	Check_Type(rb_txt, T_STRING);
	Check_Type(rb_src_enc, T_STRING);
	Check_Type(rb_dst_enc, T_STRING);

	src_txt = RSTRING_PTR(rb_txt);
	src_len = RSTRING_LEN(rb_txt);
	src_enc = RSTRING_PTR(rb_src_enc);
	dst_enc = RSTRING_PTR(rb_dst_enc);

	// first determin the size of the output buffer
	out_len = ucnv_convert(dst_enc, src_enc, NULL, 0, src_txt, src_len, &status);
	if (status != U_BUFFER_OVERFLOW_ERROR) {
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}
	out_buf = xmalloc(out_len);

	// now do the actual conversion
	status = U_ZERO_ERROR;
	out_len = ucnv_convert(dst_enc, src_enc, out_buf, out_len, src_txt, src_len, &status);
	if (!U_SUCCESS(status)) {
		xfree(out_buf);
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}

#ifdef HAVE_RUBY_ENCODING_H
	rb_enc = (void *)rb_enc_find(dst_enc);
#endif

	rb_out = charlock_new_enc_str(out_buf, out_len, rb_enc);

	xfree(out_buf);

	return rb_out;
}

// The return value needs to be freed later
static inline UChar *cstr_to_uchar(const char *src, int32_t src_len, int32_t *out_len) {
	UErrorCode status;
	UChar *dst_uchars;
	UConverter *utf8_conv;
	int32_t dst_uchar_len;

	status = U_ZERO_ERROR;
	// grab a UTF-8 converter
	utf8_conv = ucnv_open("UTF-8", &status);
	if (!U_SUCCESS(status)) {
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}

	status = U_ZERO_ERROR;
	// get the size of the buffer needed for the char -> UChar conversion
	dst_uchar_len = ucnv_toUChars(utf8_conv, NULL, 0, src, src_len, &status);
	if (status != U_BUFFER_OVERFLOW_ERROR) {
		ucnv_close(utf8_conv);
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}
	*out_len = dst_uchar_len;
	dst_uchars = xmalloc((sizeof(UChar)*dst_uchar_len)*UC_DECOMPOSITION_MAX_LENGTH);

	status = U_ZERO_ERROR;
	// actually do the char -> UChar conversion
	dst_uchar_len = ucnv_toUChars(utf8_conv, dst_uchars, dst_uchar_len, src, src_len, &status);
	if (!U_SUCCESS(status)) {
		ucnv_close(utf8_conv);
		xfree(dst_uchars);
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}

	ucnv_close(utf8_conv);
	return dst_uchars;
}

// The return value needs to be freed later
static inline char *uchar_to_cstr(UChar *src, int32_t src_len, int32_t *out_len) {
	UErrorCode status;
	UConverter *utf8_conv;
	char *dst_cstr;
	int32_t dst_cstr_len;

	status = U_ZERO_ERROR;
	// grab a UTF-8 converter
	utf8_conv = ucnv_open("UTF-8", &status);
	if (!U_SUCCESS(status)) {
		rb_raise(rb_eArgError, "%s", u_errorName(status));
	}

{'fraction_non_alphanumeric': 0.1143732860138732, 'fraction_numerical': 0.010646878528794966, 'mean_word_length': 5.746463547334058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1858860', 'n_tokens_mistral': 2632, 'n_tokens_neox': 2531, 'n_words': 547}