<commit_before>#include "mupdf/pdf.h"

fz_rect *
pdf_to_rect(fz_context *ctx, pdf_obj *array, fz_rect *r)
{
	float a = pdf_to_real(ctx, pdf_array_get(ctx, array, 0));
	float b = pdf_to_real(ctx, pdf_array_get(ctx, array, 1));
	float c = pdf_to_real(ctx, pdf_array_get(ctx, array, 2));
	float d = pdf_to_real(ctx, pdf_array_get(ctx, array, 3));
	r->x0 = fz_min(a, c);
	r->y0 = fz_min(b, d);
	r->x1 = fz_max(a, c);
	r->y1 = fz_max(b, d);
	return r;
}

fz_matrix *
pdf_to_matrix(fz_context *ctx, pdf_obj *array, fz_matrix *m)
{
	m->a = pdf_to_real(ctx, pdf_array_get(ctx, array, 0));
	m->b = pdf_to_real(ctx, pdf_array_get(ctx, array, 1));
	m->c = pdf_to_real(ctx, pdf_array_get(ctx, array, 2));
	m->d = pdf_to_real(ctx, pdf_array_get(ctx, array, 3));
	m->e = pdf_to_real(ctx, pdf_array_get(ctx, array, 4));
	m->f = pdf_to_real(ctx, pdf_array_get(ctx, array, 5));
	return m;
}

/* Convert Unicode/PdfDocEncoding string into utf-8 */
char *
pdf_to_utf8(fz_context *ctx, pdf_document *doc, pdf_obj *src)
{
	fz_buffer *strmbuf = NULL;
	unsigned char *srcptr;
	char *dstptr, *dst;
	int srclen;
	int dstlen = 0;
	int ucs;
	int i;

	fz_var(strmbuf);
	fz_try(ctx)
	{
		if (pdf_is_string(ctx, src))
		{
			srcptr = (unsigned char *) pdf_to_str_buf(ctx, src);
			srclen = pdf_to_str_len(ctx, src);
		}
		else if (pdf_is_stream(ctx, doc, pdf_to_num(ctx, src), pdf_to_gen(ctx, src)))
		{
			strmbuf = pdf_load_stream(ctx, doc, pdf_to_num(ctx, src), pdf_to_gen(ctx, src));
			srclen = fz_buffer_storage(ctx, strmbuf, (unsigned char **)&srcptr);
		}
		else
		{
			srclen = 0;
		}

		if (srclen >= 2 && srcptr[0] == 254 && srcptr[1] == 255)
		{
			for (i = 2; i + 1 < srclen; i += 2)
			{
				ucs = srcptr[i] << 8 | srcptr[i+1];
				dstlen += fz_runelen(ucs);
			}

			dstptr = dst = fz_malloc(ctx, dstlen + 1);

			for (i = 2; i + 1 < srclen; i += 2)
			{
				ucs = srcptr[i] << 8 | srcptr[i+1];
				dstptr += fz_runetochar(dstptr, ucs);
			}
		}
		else if (srclen >= 2 && srcptr[0] == 255 && srcptr[1] == 254)
		{
			for (i = 2; i + 1 < srclen; i += 2)
			{
				ucs = srcptr[i] | srcptr[i+1] << 8;
				dstlen += fz_runelen(ucs);
			}

			dstptr = dst = fz_malloc(ctx, dstlen + 1);

			for (i = 2; i + 1 < srclen; i += 2)
			{
				ucs = srcptr[i] | srcptr[i+1] << 8;
				dstptr += fz_runetochar(dstptr, ucs);
			}
		}
		else
		{
			for (i = 0; i < srclen; i++)
				dstlen += fz_runelen(pdf_doc_encoding[srcptr[i]]);

			dstptr = dst = fz_malloc(ctx, dstlen + 1);

			for (i = 0; i < srclen; i++)
			{
				ucs = pdf_doc_encoding[srcptr[i]];
				dstptr += fz_runetochar(dstptr, ucs);
			}
		}
	}
	fz_always(ctx)
	{<commit_msg>pdf: Handle surrogate pairs in pdf_to_utf8.<commit_after>#include "mupdf/pdf.h"

fz_rect *
pdf_to_rect(fz_context *ctx, pdf_obj *array, fz_rect *r)
{
	float a = pdf_to_real(ctx, pdf_array_get(ctx, array, 0));
	float b = pdf_to_real(ctx, pdf_array_get(ctx, array, 1));
	float c = pdf_to_real(ctx, pdf_array_get(ctx, array, 2));
	float d = pdf_to_real(ctx, pdf_array_get(ctx, array, 3));
	r->x0 = fz_min(a, c);
	r->y0 = fz_min(b, d);
	r->x1 = fz_max(a, c);
	r->y1 = fz_max(b, d);
	return r;
}

fz_matrix *
pdf_to_matrix(fz_context *ctx, pdf_obj *array, fz_matrix *m)
{
	m->a = pdf_to_real(ctx, pdf_array_get(ctx, array, 0));
	m->b = pdf_to_real(ctx, pdf_array_get(ctx, array, 1));
	m->c = pdf_to_real(ctx, pdf_array_get(ctx, array, 2));
	m->d = pdf_to_real(ctx, pdf_array_get(ctx, array, 3));
	m->e = pdf_to_real(ctx, pdf_array_get(ctx, array, 4));
	m->f = pdf_to_real(ctx, pdf_array_get(ctx, array, 5));
	return m;
}

static int
rune_from_utf16be(int *out, unsigned char *s, unsigned char *end)
{
	if (s + 2 <= end)
	{
		int a = s[0] << 8 | s[1];
		if (a >= 0xD800 && a <= 0xDFFF && s + 4 <= end)
		{
			int b = s[2] << 8 | s[3];
			*out = ((a - 0xD800) << 10) + (b - 0xDC00) + 0x10000;
			return 4;
		}
		*out = a;
		return 2;
	}
	*out = 0xFFFD;
	return 1;
}

/* Convert Unicode/PdfDocEncoding string into utf-8 */
char *
pdf_to_utf8(fz_context *ctx, pdf_document *doc, pdf_obj *src)
{
	fz_buffer *strmbuf = NULL;
	unsigned char *srcptr;
	char *dstptr, *dst;
	int srclen;
	int dstlen = 0;
	int ucs;
	int i;

	fz_var(strmbuf);
	fz_try(ctx)
	{
		if (pdf_is_string(ctx, src))
		{
			srcptr = (unsigned char *) pdf_to_str_buf(ctx, src);
			srclen = pdf_to_str_len(ctx, src);
		}
		else if (pdf_is_stream(ctx, doc, pdf_to_num(ctx, src), pdf_to_gen(ctx, src)))
		{
			strmbuf = pdf_load_stream(ctx, doc, pdf_to_num(ctx, src), pdf_to_gen(ctx, src));
			srclen = fz_buffer_storage(ctx, strmbuf, (unsigned char **)&srcptr);
		}
		else
		{
			srclen = 0;
		}

		/* UTF-16BE */
		if (srclen >= 2 && srcptr[0] == 254 && srcptr[1] == 255)
		{
			i = 2;
			while (i + 2 <= srclen)
			{
				/* skip language escape codes */
				if (i + 6 <= srclen &&
					srcptr[i+0] == 0 && srcptr[i+1] == 27 &&
					srcptr[i+4] == 0 && srcptr[i+5] == 27)
				{
					i += 6;
				}
				else if (i + 8 <= srclen &&
					srcptr[i+0] == 0 && srcptr[i+1] == 27 &&
					srcptr[i+6] == 0 && srcptr[i+7] == 27)
				{
					i += 8;
				}
				else
				{
					i += rune_from_utf16be(&ucs, srcptr + i, srcptr + srclen);
					dstlen += fz_runelen(ucs);
				}
			}

			dstptr = dst = fz_malloc(ctx, dstlen + 1);

			i = 2;
			while (i + 2 <= srclen)
			{
				/* skip language escape codes */
				if (i + 6 <= srclen &&
					srcptr[i+0] == 0 && srcptr[i+1] == 27 &&
					srcptr[i+4] == 0 && srcptr[i+5] == 27)
				{
					i += 6;
				}
				else if (i + 8 <= srclen &&
					srcptr[i+0] == 0 && srcptr[i+1] == 27 &&
					srcptr[i+6] == 0 && srcptr[i+7] == 27)
				{
					i += 8;
				}
				else
				{
					i += rune_from_utf16be(&ucs, srcptr + i, srcptr + srclen);
					dstptr += fz_runetochar(dstptr, ucs);
				}
			}
		}

		/* PDFDocEncoding */
		else
		{
			for (i = 0; i < srclen; i++)
				dstlen += fz_runelen(pdf_doc_encoding[srcptr[i]]);

			dstptr = dst = fz_malloc(ctx, dstlen + 1);

			for (i = 0; i < srclen; i++)
			{
				ucs = pdf_doc_encoding[srcptr[i]];
				dstptr += fz_runetochar(dstptr, ucs);
			}
		}
	}
	fz_always(ctx)
	{
{'fraction_non_alphanumeric': 0.1830305050841807, 'fraction_numerical': 0.030838473078846473, 'mean_word_length': 3.1095890410958904, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7424118', 'n_tokens_mistral': 3259, 'n_tokens_neox': 2785, 'n_words': 606}