<commit_before>#ifdef STATIC
/* Pre-boot environment: included */

/* prevent inclusion of _LINUX_KERNEL_H in pre-boot environment: lots
 * errors about console_printk etc... on ARM */
#define _LINUX_KERNEL_H

#include "zlib_inflate/inftrees.c"
#include "zlib_inflate/inffast.c"
#include "zlib_inflate/inflate.c"

#else /* STATIC */
/* initramfs et al: linked */

#include <linux/zutil.h>

#include "zlib_inflate/inftrees.h"
#include "zlib_inflate/inffast.h"
#include "zlib_inflate/inflate.h"

#include "zlib_inflate/infutil.h"

#endif /* STATIC */

#include <linux/decompress/mm.h>

#define INBUF_LEN (16*1024)

/* Included from initramfs et al code */
STATIC int INIT gunzip(unsigned char *buf, int len,
		       int(*fill)(void*, unsigned int),
		       int(*flush)(void*, unsigned int),
		       unsigned char *out_buf,
		       int *pos,
		       void(*error_fn)(char *x)) {
	u8 *zbuf;
	struct z_stream_s *strm;
	int rc;
	size_t out_len;

	set_error_fn(error_fn);
	rc = -1;
	if (flush) {
		out_len = 0x8100; /* 32 K */
		out_buf = malloc(out_len);
	} else {
		out_len = 0x7fffffff; /* no limit */
	}
	if (!out_buf) {
		error("Out of memory while allocating output buffer");
		goto gunzip_nomem1;
	}

	if (buf)
		zbuf = buf;
	else {
		zbuf = malloc(INBUF_LEN);
		len = 0;
	}
	if (!zbuf) {
		error("Out of memory while allocating input buffer");
		goto gunzip_nomem2;
	}

	strm = malloc(sizeof(*strm));
	if (strm == NULL) {
		error("Out of memory while allocating z_stream");
		goto gunzip_nomem3;
	}

	strm->workspace = malloc(flush ? zlib_inflate_workspacesize() :
				 sizeof(struct inflate_state));
	if (strm->workspace == NULL) {
		error("Out of memory while allocating workspace");
		goto gunzip_nomem4;
	}

	if (len == 0)
		len = fill(zbuf, INBUF_LEN);

	/* verify the gzip header */
	if (len < 10 ||
	   zbuf[0] != 0x1f || zbuf[1] != 0x8b || zbuf[2] != 0x08) {
		if (pos)
			*pos = 0;
		error("Not a gzip file");
		goto gunzip_5;
	}

	/* skip over gzip header (1f,8b,08... 10 bytes total +
	 * possible asciz filename)
	 */
	strm->next_in = zbuf + 10;
	/* skip over asciz filename */
	if (zbuf[3] & 0x8) {
		while (strm->next_in[0])
			strm->next_in++;
		strm->next_in++;
	}
	strm->avail_in = len - 10;

	strm->next_out = out_buf;
	strm->avail_out = out_len;

	rc = zlib_inflateInit2(strm, -MAX_WBITS);

	if (!flush) {
		WS(strm)->inflate_state.wsize = 0;
		WS(strm)->inflate_state.window = NULL;
	}

	while (rc == Z_OK) {
		if (strm->avail_in == 0) {
			/* TODO: handle case where both pos and fill are set */
			len = fill(zbuf, INBUF_LEN);
			if (len < 0) {
				rc = -1;
				error("read error");
				break;
			}
			strm->next_in = zbuf;
			strm->avail_in = len;
		}
		rc = zlib_inflate(strm, 0);

		/* Write any data generated */
		if (flush && strm->next_out > out_buf) {
			int l = strm->next_out - out_buf;
			if (l != flush(out_buf, l)) {
				rc = -1;
				error("write error");
				break;
			}
			strm->next_out = out_buf;
			strm->avail_out = out_len;
		}

		/* after Z_FINISH, only Z_STREAM_END is "we unpacked it all" */
		if (rc == Z_STREAM_END) {
			rc = 0;
			break;
		} else if (rc != Z_OK) {
			error("uncompression error");
			rc = -1;
		}
	}

	zlib_inflateEnd(strm);
	if (pos)
		/* add + 8 to skip over trailer */
		*pos = strm->next_in - zbuf+8;

gunzip_5:
	free(strm->workspace);
gunzip_nomem4:
	free(strm);
gunzip_nomem3:
	if (!buf)
		free(zbuf);
gunzip_nomem2:
	if (flush)
		free(out_buf);
gunzip_nomem1:
	return rc; /* returns Z_OK (0) if successful */
}

#define decompress gunzip
<commit_msg>bzip2/lzma: fix constant in decompress_inflate<commit_after>#ifdef STATIC
/* Pre-boot environment: included */

/* prevent inclusion of _LINUX_KERNEL_H in pre-boot environment: lots
 * errors about console_printk etc... on ARM */
#define _LINUX_KERNEL_H

#include "zlib_inflate/inftrees.c"
#include "zlib_inflate/inffast.c"
#include "zlib_inflate/inflate.c"

#else /* STATIC */
/* initramfs et al: linked */

#include <linux/zutil.h>

#include "zlib_inflate/inftrees.h"
#include "zlib_inflate/inffast.h"
#include "zlib_inflate/inflate.h"

#include "zlib_inflate/infutil.h"

#endif /* STATIC */

#include <linux/decompress/mm.h>

#define INBUF_LEN (16*1024)

/* Included from initramfs et al code */
STATIC int INIT gunzip(unsigned char *buf, int len,
		       int(*fill)(void*, unsigned int),
		       int(*flush)(void*, unsigned int),
		       unsigned char *out_buf,
		       int *pos,
		       void(*error_fn)(char *x)) {
	u8 *zbuf;
	struct z_stream_s *strm;
	int rc;
	size_t out_len;

	set_error_fn(error_fn);
	rc = -1;
	if (flush) {
		out_len = 0x8000; /* 32 K */
		out_buf = malloc(out_len);
	} else {
		out_len = 0x7fffffff; /* no limit */
	}
	if (!out_buf) {
		error("Out of memory while allocating output buffer");
		goto gunzip_nomem1;
	}

	if (buf)
		zbuf = buf;
	else {
		zbuf = malloc(INBUF_LEN);
		len = 0;
	}
	if (!zbuf) {
		error("Out of memory while allocating input buffer");
		goto gunzip_nomem2;
	}

	strm = malloc(sizeof(*strm));
	if (strm == NULL) {
		error("Out of memory while allocating z_stream");
		goto gunzip_nomem3;
	}

	strm->workspace = malloc(flush ? zlib_inflate_workspacesize() :
				 sizeof(struct inflate_state));
	if (strm->workspace == NULL) {
		error("Out of memory while allocating workspace");
		goto gunzip_nomem4;
	}

	if (len == 0)
		len = fill(zbuf, INBUF_LEN);

	/* verify the gzip header */
	if (len < 10 ||
	   zbuf[0] != 0x1f || zbuf[1] != 0x8b || zbuf[2] != 0x08) {
		if (pos)
			*pos = 0;
		error("Not a gzip file");
		goto gunzip_5;
	}

	/* skip over gzip header (1f,8b,08... 10 bytes total +
	 * possible asciz filename)
	 */
	strm->next_in = zbuf + 10;
	/* skip over asciz filename */
	if (zbuf[3] & 0x8) {
		while (strm->next_in[0])
			strm->next_in++;
		strm->next_in++;
	}
	strm->avail_in = len - 10;

	strm->next_out = out_buf;
	strm->avail_out = out_len;

	rc = zlib_inflateInit2(strm, -MAX_WBITS);

	if (!flush) {
		WS(strm)->inflate_state.wsize = 0;
		WS(strm)->inflate_state.window = NULL;
	}

	while (rc == Z_OK) {
		if (strm->avail_in == 0) {
			/* TODO: handle case where both pos and fill are set */
			len = fill(zbuf, INBUF_LEN);
			if (len < 0) {
				rc = -1;
				error("read error");
				break;
			}
			strm->next_in = zbuf;
			strm->avail_in = len;
		}
		rc = zlib_inflate(strm, 0);

		/* Write any data generated */
		if (flush && strm->next_out > out_buf) {
			int l = strm->next_out - out_buf;
			if (l != flush(out_buf, l)) {
				rc = -1;
				error("write error");
				break;
			}
			strm->next_out = out_buf;
			strm->avail_out = out_len;
		}

		/* after Z_FINISH, only Z_STREAM_END is "we unpacked it all" */
		if (rc == Z_STREAM_END) {
			rc = 0;
			break;
		} else if (rc != Z_OK) {
			error("uncompression error");
			rc = -1;
		}
	}

	zlib_inflateEnd(strm);
	if (pos)
		/* add + 8 to skip over trailer */
		*pos = strm->next_in - zbuf+8;

gunzip_5:
	free(strm->workspace);
gunzip_nomem4:
	free(strm);
gunzip_nomem3:
	if (!buf)
		free(zbuf);
gunzip_nomem2:
	if (flush)
		free(out_buf);
gunzip_nomem1:
	return rc; /* returns Z_OK (0) if successful */
}

#define decompress gunzip

{'fraction_non_alphanumeric': 0.1526739099760124, 'fraction_numerical': 0.019331169747424864, 'mean_word_length': 3.189125295508274, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '481635', 'n_tokens_mistral': 3220, 'n_tokens_neox': 2902, 'n_words': 702}