<commit_before>#include "spritepack.h"
#include "matrix.h"
#include "shader.h"

#include <lua.h>
#include <lauxlib.h>

#include <stdint.h>
#include <string.h>

#define TAG_ID 1
#define TAG_COLOR 2
#define TAG_ADDITIVE 4
#define TAG_MATRIX 8
#define TAG_TOUCH 16

struct import_alloc {
	lua_State *L;
	char * buffer;
	int cap;
};

static void *
ialloc(struct import_alloc *alloc, int size) {
	if (alloc->cap < size) {
		luaL_error(alloc->L, "import invalid stream, alloc failed");
	}
	void * ret = alloc->buffer;
	alloc->buffer += size;
	alloc->cap -= size;
	return ret;
}

struct import_stream {
	struct import_alloc *alloc;
	struct sprite_pack *pack;
	const char * stream;
	size_t size;
	// for debug
	int current_id;
};

static int
import_byte(struct import_stream *is) {
	if (is->size == 0) {
		luaL_error(is->alloc->L, "Invalid import stream (%d)", is->current_id);
	}
	uint8_t c = (uint8_t)*(is->stream);
	++is->stream;
	--is->size;

	return c;
}

static int
import_word(struct import_stream *is) {
	if (is->size < 2) {
		luaL_error(is->alloc->L, "Invalid import stream (%d)", is->current_id);
	}
	uint8_t low = (uint8_t)*(is->stream);
	uint8_t high = (uint8_t)*(is->stream+1);
	is->stream += 2;
	is->size -= 2;

	return low | (uint32_t)high << 8;
}

static int32_t
import_int32(struct import_stream *is) {
	if (is->size < 4) {
		luaL_error(is->alloc->L, "Invalid import stream (%d)",is->current_id);
	}
	uint8_t b[4];
	b[0] = (uint8_t)*(is->stream);
	b[1] = (uint8_t)*(is->stream+1);
	b[2] = (uint8_t)*(is->stream+2);
	b[3] = (uint8_t)*(is->stream+3);
	is->stream += 4;
	is->size -= 4;

	return (int32_t)(b[0] | (uint32_t)b[1]<<8 | (uint32_t)b[2]<<16 | (uint32_t)b[3]<<24);
}

static uint32_t
import_color(struct import_stream *is) {
	if (is->size < 4) {
		luaL_error(is->alloc->L, "Invalid import stream (%d) where import color", is->current_id);
	}
	uint8_t b[4];
	b[0] = (uint8_t)*(is->stream);
	b[1] = (uint8_t)*(is->stream+1);
	b[2] = (uint8_t)*(is->stream+2);
	b[3] = (uint8_t)*(is->stream+3);
	is->stream += 4;
	is->size -= 4;

	return (b[0] | (uint32_t)b[1]<<8 | (uint32_t)b[2]<<16 | (uint32_t)b[3]<<24);
}

static void
import_picture(struct import_stream *is) {
	int n = import_byte(is);
	struct pack_picture * pp = (struct pack_picture *)ialloc(is->alloc, sizeof(*pp) + (n - 1) * sizeof(struct pack_quad));
	pp->n = n;
	int i,j;
	for (i=0;i<n;i++) {
		struct pack_quad * q = &pp->rect[i];
		int texid = import_byte(is);
		q->texid = is->pack->tex[texid];
		for (j=0;j<8;j++) {
			q->texture_coord[j] = import_word(is);
		}
		for (j=0;j<8;j++) {
			q->screen_coord[j] = import_int32(is);
		}
	}
}

static void
import_polygon(struct import_stream *is) {
	int n = import_byte(is);
	struct pack_polygon * pp = (struct pack_polygon *)ialloc(is->alloc, sizeof(*pp) + (n - 1) * sizeof(struct pack_poly));
	pp->n = n;
	int i,j;
	for (i=0;i<n;i++) {
		struct pack_poly * p = &pp->poly[i];
		int texid = import_byte(is);
		p->texid = is->pack->tex[texid];
		p->n = import_byte(is);
		p->texture_coord = (uint16_t *)ialloc(is->alloc, p->n * 2 * sizeof(uint16_t));
		p->screen_coord = (int32_t *)ialloc(is->alloc, p->n * 2 * sizeof(uint32_t));
		for (j=0;j<p->n*2;j++) {
			p->texture_coord[j] = import_word(is);
		}
		for (j=0;j<p->n*2;j++) {
			p->screen_coord[j] = import_int32(is);
		}
	}
}

static const char *
import_string(struct import_stream *is) {
	int n = import_byte(is);
	if (n==255) {
		return NULL;
	}
	if (is->size < n) {
		luaL_error(is->alloc->L, "Invalid stream (%d): read string failed", is->current_id);
	}
	char * buf = (char *)ialloc(is->alloc, (n+1+3) & ~3);
	memcpy(buf, is->stream, n);
	buf[n] = 0;
	is->stream += n;
	is->size -= n;

	return buf;
}

static void
import_frame(struct pack_frame * pf, struct import_stream *is, int maxc) {
	int n = import_byte(is);
	int i;
	pf->part = (struct pack_part *)ialloc(is->alloc, n * sizeof(struct pack_part));
	pf->n = n;
	for (i=0;i<n;i++) {
		struct pack_part *pp = &pf->part[i];
		int tag = import_byte(is);
		if (tag & TAG_ID) {
			pp->component_id = import_word(is);
			if (pp->component_id >= maxc) {
				luaL_error(is->alloc->L, "Invalid stream (%d): frame part has wrong component id", is->current_id);
			}
		} else {
			luaL_error(is->alloc->L, "Invalid stream (%d): frame part need an id", is->current_id);
		}
		if (tag & TAG_MATRIX) {
			pp->t.mat = (struct matrix *)ialloc(is->alloc, sizeof(struct matrix));
			int32_t *m = pp->t.mat->m;
			int j;
			for (j=0;j<6;j++) {
				m[j] = import_int32(is);
			}
		} else {
			pp->t.mat = NULL;
		}
		if (tag & TAG_COLOR) {
			pp->t.color = import_color(is);
		} else {
			pp->t.color = 0xffffffff;<commit_msg>components limit in a single frame raise to 0xffff<commit_after>#include "spritepack.h"
#include "matrix.h"
#include "shader.h"

#include <lua.h>
#include <lauxlib.h>

#include <stdint.h>
#include <string.h>

#define TAG_ID 1
#define TAG_COLOR 2
#define TAG_ADDITIVE 4
#define TAG_MATRIX 8
#define TAG_TOUCH 16

struct import_alloc {
	lua_State *L;
	char * buffer;
	int cap;
};

static void *
ialloc(struct import_alloc *alloc, int size) {
	if (alloc->cap < size) {
		luaL_error(alloc->L, "import invalid stream, alloc failed");
	}
	void * ret = alloc->buffer;
	alloc->buffer += size;
	alloc->cap -= size;
	return ret;
}

struct import_stream {
	struct import_alloc *alloc;
	struct sprite_pack *pack;
	const char * stream;
	size_t size;
	// for debug
	int current_id;
};

static int
import_byte(struct import_stream *is) {
	if (is->size == 0) {
		luaL_error(is->alloc->L, "Invalid import stream (%d)", is->current_id);
	}
	uint8_t c = (uint8_t)*(is->stream);
	++is->stream;
	--is->size;

	return c;
}

static int
import_word(struct import_stream *is) {
	if (is->size < 2) {
		luaL_error(is->alloc->L, "Invalid import stream (%d)", is->current_id);
	}
	uint8_t low = (uint8_t)*(is->stream);
	uint8_t high = (uint8_t)*(is->stream+1);
	is->stream += 2;
	is->size -= 2;

	return low | (uint32_t)high << 8;
}

static int32_t
import_int32(struct import_stream *is) {
	if (is->size < 4) {
		luaL_error(is->alloc->L, "Invalid import stream (%d)",is->current_id);
	}
	uint8_t b[4];
	b[0] = (uint8_t)*(is->stream);
	b[1] = (uint8_t)*(is->stream+1);
	b[2] = (uint8_t)*(is->stream+2);
	b[3] = (uint8_t)*(is->stream+3);
	is->stream += 4;
	is->size -= 4;

	return (int32_t)(b[0] | (uint32_t)b[1]<<8 | (uint32_t)b[2]<<16 | (uint32_t)b[3]<<24);
}

static uint32_t
import_color(struct import_stream *is) {
	if (is->size < 4) {
		luaL_error(is->alloc->L, "Invalid import stream (%d) where import color", is->current_id);
	}
	uint8_t b[4];
	b[0] = (uint8_t)*(is->stream);
	b[1] = (uint8_t)*(is->stream+1);
	b[2] = (uint8_t)*(is->stream+2);
	b[3] = (uint8_t)*(is->stream+3);
	is->stream += 4;
	is->size -= 4;

	return (b[0] | (uint32_t)b[1]<<8 | (uint32_t)b[2]<<16 | (uint32_t)b[3]<<24);
}

static void
import_picture(struct import_stream *is) {
	int n = import_byte(is);
	struct pack_picture * pp = (struct pack_picture *)ialloc(is->alloc, sizeof(*pp) + (n - 1) * sizeof(struct pack_quad));
	pp->n = n;
	int i,j;
	for (i=0;i<n;i++) {
		struct pack_quad * q = &pp->rect[i];
		int texid = import_byte(is);
		q->texid = is->pack->tex[texid];
		for (j=0;j<8;j++) {
			q->texture_coord[j] = import_word(is);
		}
		for (j=0;j<8;j++) {
			q->screen_coord[j] = import_int32(is);
		}
	}
}

static void
import_polygon(struct import_stream *is) {
	int n = import_byte(is);
	struct pack_polygon * pp = (struct pack_polygon *)ialloc(is->alloc, sizeof(*pp) + (n - 1) * sizeof(struct pack_poly));
	pp->n = n;
	int i,j;
	for (i=0;i<n;i++) {
		struct pack_poly * p = &pp->poly[i];
		int texid = import_byte(is);
		p->texid = is->pack->tex[texid];
		p->n = import_byte(is);
		p->texture_coord = (uint16_t *)ialloc(is->alloc, p->n * 2 * sizeof(uint16_t));
		p->screen_coord = (int32_t *)ialloc(is->alloc, p->n * 2 * sizeof(uint32_t));
		for (j=0;j<p->n*2;j++) {
			p->texture_coord[j] = import_word(is);
		}
		for (j=0;j<p->n*2;j++) {
			p->screen_coord[j] = import_int32(is);
		}
	}
}

static const char *
import_string(struct import_stream *is) {
	int n = import_byte(is);
	if (n==255) {
		return NULL;
	}
	if (is->size < n) {
		luaL_error(is->alloc->L, "Invalid stream (%d): read string failed", is->current_id);
	}
	char * buf = (char *)ialloc(is->alloc, (n+1+3) & ~3);
	memcpy(buf, is->stream, n);
	buf[n] = 0;
	is->stream += n;
	is->size -= n;

	return buf;
}

static void
import_frame(struct pack_frame * pf, struct import_stream *is, int maxc) {
	int n = import_word(is);
	int i;
	pf->part = (struct pack_part *)ialloc(is->alloc, n * sizeof(struct pack_part));
	pf->n = n;
	for (i=0;i<n;i++) {
		struct pack_part *pp = &pf->part[i];
		int tag = import_byte(is);
		if (tag & TAG_ID) {
			pp->component_id = import_word(is);
			if (pp->component_id >= maxc) {
				luaL_error(is->alloc->L, "Invalid stream (%d): frame part has wrong component id", is->current_id);
			}
		} else {
			luaL_error(is->alloc->L, "Invalid stream (%d): frame part need an id", is->current_id);
		}
		if (tag & TAG_MATRIX) {
			pp->t.mat = (struct matrix *)ialloc(is->alloc, sizeof(struct matrix));
			int32_t *m = pp->t.mat->m;
			int j;
			for (j=0;j<6;j++) {
				m[j] = import_int32(is);
			}
		} else {
			pp->t.mat = NULL;
		}
		if (tag & TAG_COLOR) {
			pp->t.color = import_color(is);
		} else {
			pp->t.color = 0xffffffff;
{'fraction_non_alphanumeric': 0.20423064533447868, 'fraction_numerical': 0.028240094491570923, 'mean_word_length': 4.031874662344679, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 197, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '716547', 'n_tokens_mistral': 4319, 'n_tokens_neox': 4026, 'n_words': 861}