<commit_before>/*
 * Miscellaneous I/O functions
 */

#include "fitz-base.h"
#include "fitz-stream.h"

int
fz_tell(fz_stream *stm)
{
	if (stm->mode == FZ_SREAD)
		return fz_rtell(stm);
	return fz_wtell(stm);
}

fz_error *
fz_seek(fz_stream *stm, int offset, int whence)
{
	if (stm->mode == FZ_SREAD)
		return fz_rseek(stm, offset, whence);
	return fz_wseek(stm, offset, whence);
}

/*
 * Read a line terminated by LF or CR or CRLF.
 */

fz_error *
fz_readline(fz_stream *stm, char *mem, int n)
{
	fz_error *error;

	char *s = mem;
	int c = EOF;
	while (n > 1)
	{
		c = fz_readbyte(stm);
		if (c == EOF)
			break;
		if (c == '\r') {
			c = fz_peekbyte(stm);
			if (c == '\n')
				c = fz_readbyte(stm);
			break;
		}
		if (c == '\n')
			break;
		*s++ = c;
		n--;
	}
	if (n)
		*s = '\0';

	error = fz_readerror(stm);
	if (error)
		return fz_rethrow(error, "cannot read line");
	return fz_okay;
}

/*
 * Utility function to consume all the contents of an input stream into
 * a freshly allocated buffer; realloced and trimmed to size.
 */

enum { CHUNKSIZE = 1024 * 4 };

fz_error *
fz_readall(fz_buffer **bufp, fz_stream *stm)
{
	fz_error *error;
	fz_buffer *real;
	unsigned char *newbuf;
	unsigned char *buf;
	int len;
	int pos;
	int n;

	len = 0;
	pos = 0;
	buf = nil;

	while (1)
	{
		if (len - pos == 0)
		{
			len += CHUNKSIZE;
			newbuf = fz_realloc(buf, len);
			if (!newbuf)
			{
				fz_free(buf);
				return fz_throw("outofmem: scratch buffer");
			}
			buf = newbuf;
		}

		error = fz_read(&n, stm, buf + pos, len - pos);
		if (error)
		{
			fz_free(buf);
			return fz_rethrow(error, "cannot read data");
		}

		pos += n;

		if (n < CHUNKSIZE)
		{
			if (pos > 0)
			{
				newbuf = fz_realloc(buf, pos);
				if (!newbuf)
				{
					fz_free(buf);
					return fz_throw("outofmem: scratch buffer");
				}<commit_msg>Double the buffer size each time it needs to grow instead of incrementing it by a small value. This fixes a lot of performance issues when loading images.<commit_after>/*
 * Miscellaneous I/O functions
 */

#include "fitz-base.h"
#include "fitz-stream.h"

int
fz_tell(fz_stream *stm)
{
	if (stm->mode == FZ_SREAD)
		return fz_rtell(stm);
	return fz_wtell(stm);
}

fz_error *
fz_seek(fz_stream *stm, int offset, int whence)
{
	if (stm->mode == FZ_SREAD)
		return fz_rseek(stm, offset, whence);
	return fz_wseek(stm, offset, whence);
}

/*
 * Read a line terminated by LF or CR or CRLF.
 */

fz_error *
fz_readline(fz_stream *stm, char *mem, int n)
{
	fz_error *error;

	char *s = mem;
	int c = EOF;
	while (n > 1)
	{
		c = fz_readbyte(stm);
		if (c == EOF)
			break;
		if (c == '\r') {
			c = fz_peekbyte(stm);
			if (c == '\n')
				c = fz_readbyte(stm);
			break;
		}
		if (c == '\n')
			break;
		*s++ = c;
		n--;
	}
	if (n)
		*s = '\0';

	error = fz_readerror(stm);
	if (error)
		return fz_rethrow(error, "cannot read line");
	return fz_okay;
}

/*
 * Utility function to consume all the contents of an input stream into
 * a freshly allocated buffer; realloced and trimmed to size.
 */

enum { CHUNKSIZE = 1024 * 4 };

fz_error *
fz_readall(fz_buffer **bufp, fz_stream *stm)
{
	fz_error *error;
	fz_buffer *real;
	unsigned char *newbuf;
	unsigned char *buf;
	int len;
	int pos;
	int n;

	len = 0;
	pos = 0;
	buf = nil;

	while (1)
	{
		if (len - pos == 0)
		{
			if (len == 0)
				len = CHUNKSIZE;
			else
				len = len * 2;
			newbuf = fz_realloc(buf, len);
			if (!newbuf)
			{
				fz_free(buf);
				return fz_throw("outofmem: scratch buffer");
			}
			buf = newbuf;
		}

		error = fz_read(&n, stm, buf + pos, len - pos);
		if (error)
		{
			fz_free(buf);
			return fz_rethrow(error, "cannot read data");
		}

		pos += n;

		if (n < CHUNKSIZE)
		{
			if (pos > 0)
			{
				newbuf = fz_realloc(buf, pos);
				if (!newbuf)
				{
					fz_free(buf);
					return fz_throw("outofmem: scratch buffer");
				}
{'fraction_non_alphanumeric': 0.15006535947712418, 'fraction_numerical': 0.0067973856209150325, 'mean_word_length': 2.8529707955689827, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3467106', 'n_tokens_mistral': 1900, 'n_tokens_neox': 1675, 'n_words': 454}