<commit_before>#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <lua.h>
#include <lauxlib.h>

#include "silly.h"
#include "silly_log.h"
#include "silly_socket.h"
#include "silly_malloc.h"

#define min(a, b) ((a) < (b) ? (a) : (b))

#define POOL (lua_upvalueindex(1))
#define NB (1)

//2^4=16KiB
#define CHUNK (1 << 4)
//2^11=2KiB
#define CHUNK_EXP (11 - 4)

struct node {
	int size;
	char *buff;
	struct node *next;
};

struct node_buffer {
	int sid;
	int size;
	int offset;
	int limit;
	struct node *head;
	struct node **tail;
};

static int
free_pool(lua_State *L)
{
	struct node *n, *end;
	n = (struct node *)luaL_checkudata(L, 1, "nodepool");
	int sz = lua_rawlen(L, 1);
	assert(sz % sizeof(struct node) == 0);
	sz /= sizeof(struct node);
	end = n + sz;
	while (n < end) {
		if (n->buff) {
			silly_free(n->buff);
			n->buff = NULL;
		}
		++n;
	}
	return 0;
}

static struct node *
new_pool(lua_State *L, int n)
{
	int i;
	struct node *free;
	free = (struct node *)lua_newuserdata(L, sizeof(struct node) * n);
	memset(free, 0, sizeof(struct node) * n);
	for (i = 0; i < n - 1; i++)
		free[i].next = &free[i + 1];
	free[i].next = NULL;
	if (luaL_newmetatable(L, "nodepool")) {
		lua_pushcfunction(L, free_pool);
		lua_setfield(L, -2, "__gc");
	}
	lua_setmetatable(L, -2);
	return free;
}

static struct node *
new_node(lua_State *L)
{
	lua_rawgeti(L, POOL, 1);
	struct node *free;
	if (lua_isnil(L, -1)) {
		int sz, exp;
		struct node *new;
		int n = lua_rawlen(L, POOL);
		if (n == 0)
			n = 1;
		exp = n > CHUNK_EXP ? CHUNK_EXP : n;
		sz = CHUNK << exp;
		new = new_pool(L, sz);
		lua_rawseti(L, POOL, n + 1);
		free = new;
	} else {
		free = (struct node *)lua_touserdata(L, -1);
	}
	lua_pop(L, 1);
	if (free->next)
		lua_pushlightuserdata(L, free->next);
	else
		lua_pushnil(L);
	lua_rawseti(L, POOL, 1);
	return free;
}


static inline void
append_node(struct node_buffer *nb, struct node *n)
{
	n->next = NULL;
	*nb->tail = n;
	nb->tail = &n->next;
	nb->size += n->size;
	return ;
}

//'remove_node'may be called by 'lfree' which may be called in
//'nodebuffer'.__gc, and 'nodebuffer'.__gc may called before
//'nodepool'.__gc, so 'nodebuffer'.__gc should set the n->buff as 'null'
//after it free n->buff. In lua GC circle, one circle call __gc function
//and next circle really clear the memory of object,
//so even 'nodebuffer'.__gc free the n->buff, the pointer 'n' is sitll valid

static void
remove_node(lua_State *L, struct node_buffer *nb, struct node *n)
{
	assert(n == nb->head);
	assert(nb->tail);
	nb->head = nb->head->next;
	nb->offset = 0;
	if (nb->head == NULL)
		nb->tail = &nb->head;
	silly_free(n->buff);
	n->buff = NULL;
	lua_rawgeti(L, POOL, 1);
	struct node *free = lua_touserdata(L, -1);
	if (free == NULL) {
		n->next = NULL;
		lua_pushlightuserdata(L, n);
		lua_rawseti(L, POOL, 1);
	} else {
		n->next = free->next;
		free->next = n;
	}
	lua_pop(L, 1);
	return ;
}

static int
lfree(lua_State *L)
{
	struct node_buffer *nb;
	if (lua_isnil(L, NB))
		return 0;
	nb = (struct node_buffer *)luaL_checkudata(L, NB, "nodebuffer");
	nb->size = 0;
	while (nb->head)
		remove_node(L, nb, nb->head);
	return 0;
}

static int
lnew(lua_State *L)
{
	struct node_buffer *nb;
	nb = (struct node_buffer *)lua_newuserdata(L, sizeof(*nb));
	nb->sid = luaL_checkinteger(L, 1);
	nb->offset = 0;
	nb->size = 0;
	nb->head = NULL;
	nb->limit = 32*1024*1024;
	nb->tail = &nb->head;
	if (luaL_newmetatable(L, "nodebuffer")) {
		lua_pushvalue(L, POOL);
		lua_pushcclosure(L, lfree, 1);
		lua_setfield(L, -2, "__gc");
	}
	lua_setmetatable(L, -2);
	return 1;
}


//@input
//	node, silly_message_socket
//@return
//	node buffer

static int
push(lua_State *L, int sid, char *data, int sz)
{
	int oldsize, limit;
	struct node_buffer *nb;
	struct node *new = new_node(L);
	new->size = sz;
	new->buff = data;
	nb = (struct node_buffer *)luaL_checkudata(L, NB, "nodebuffer");
	assert(nb->sid == sid);
	limit = nb->limit;
	oldsize = nb->size;
	append_node(nb, new);
	if (oldsize < limit && nb->size >= limit)
		silly_socket_readctrl(nb->sid, 0);<commit_msg>netstream default buffer limit to INT_MAX<commit_after>#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <lua.h>
#include <lauxlib.h>

#include "silly.h"
#include "silly_log.h"
#include "silly_socket.h"
#include "silly_malloc.h"

#define min(a, b) ((a) < (b) ? (a) : (b))

#define POOL (lua_upvalueindex(1))
#define NB (1)

//2^4=16KiB
#define CHUNK (1 << 4)
//2^11=2KiB
#define CHUNK_EXP (11 - 4)

struct node {
	int size;
	char *buff;
	struct node *next;
};

struct node_buffer {
	int sid;
	int size;
	int offset;
	int limit;
	struct node *head;
	struct node **tail;
};

static int
free_pool(lua_State *L)
{
	struct node *n, *end;
	n = (struct node *)luaL_checkudata(L, 1, "nodepool");
	int sz = lua_rawlen(L, 1);
	assert(sz % sizeof(struct node) == 0);
	sz /= sizeof(struct node);
	end = n + sz;
	while (n < end) {
		if (n->buff) {
			silly_free(n->buff);
			n->buff = NULL;
		}
		++n;
	}
	return 0;
}

static struct node *
new_pool(lua_State *L, int n)
{
	int i;
	struct node *free;
	free = (struct node *)lua_newuserdata(L, sizeof(struct node) * n);
	memset(free, 0, sizeof(struct node) * n);
	for (i = 0; i < n - 1; i++)
		free[i].next = &free[i + 1];
	free[i].next = NULL;
	if (luaL_newmetatable(L, "nodepool")) {
		lua_pushcfunction(L, free_pool);
		lua_setfield(L, -2, "__gc");
	}
	lua_setmetatable(L, -2);
	return free;
}

static struct node *
new_node(lua_State *L)
{
	lua_rawgeti(L, POOL, 1);
	struct node *free;
	if (lua_isnil(L, -1)) {
		int sz, exp;
		struct node *new;
		int n = lua_rawlen(L, POOL);
		if (n == 0)
			n = 1;
		exp = n > CHUNK_EXP ? CHUNK_EXP : n;
		sz = CHUNK << exp;
		new = new_pool(L, sz);
		lua_rawseti(L, POOL, n + 1);
		free = new;
	} else {
		free = (struct node *)lua_touserdata(L, -1);
	}
	lua_pop(L, 1);
	if (free->next)
		lua_pushlightuserdata(L, free->next);
	else
		lua_pushnil(L);
	lua_rawseti(L, POOL, 1);
	return free;
}


static inline void
append_node(struct node_buffer *nb, struct node *n)
{
	n->next = NULL;
	*nb->tail = n;
	nb->tail = &n->next;
	nb->size += n->size;
	return ;
}

//'remove_node'may be called by 'lfree' which may be called in
//'nodebuffer'.__gc, and 'nodebuffer'.__gc may called before
//'nodepool'.__gc, so 'nodebuffer'.__gc should set the n->buff as 'null'
//after it free n->buff. In lua GC circle, one circle call __gc function
//and next circle really clear the memory of object,
//so even 'nodebuffer'.__gc free the n->buff, the pointer 'n' is sitll valid

static void
remove_node(lua_State *L, struct node_buffer *nb, struct node *n)
{
	assert(n == nb->head);
	assert(nb->tail);
	nb->head = nb->head->next;
	nb->offset = 0;
	if (nb->head == NULL)
		nb->tail = &nb->head;
	silly_free(n->buff);
	n->buff = NULL;
	lua_rawgeti(L, POOL, 1);
	struct node *free = lua_touserdata(L, -1);
	if (free == NULL) {
		n->next = NULL;
		lua_pushlightuserdata(L, n);
		lua_rawseti(L, POOL, 1);
	} else {
		n->next = free->next;
		free->next = n;
	}
	lua_pop(L, 1);
	return ;
}

static int
lfree(lua_State *L)
{
	struct node_buffer *nb;
	if (lua_isnil(L, NB))
		return 0;
	nb = (struct node_buffer *)luaL_checkudata(L, NB, "nodebuffer");
	nb->size = 0;
	while (nb->head)
		remove_node(L, nb, nb->head);
	return 0;
}

static int
lnew(lua_State *L)
{
	struct node_buffer *nb;
	nb = (struct node_buffer *)lua_newuserdata(L, sizeof(*nb));
	nb->sid = luaL_checkinteger(L, 1);
	nb->offset = 0;
	nb->size = 0;
	nb->head = NULL;
	nb->limit = INT_MAX;
	nb->tail = &nb->head;
	if (luaL_newmetatable(L, "nodebuffer")) {
		lua_pushvalue(L, POOL);
		lua_pushcclosure(L, lfree, 1);
		lua_setfield(L, -2, "__gc");
	}
	lua_setmetatable(L, -2);
	return 1;
}


//@input
//	node, silly_message_socket
//@return
//	node buffer

static int
push(lua_State *L, int sid, char *data, int sz)
{
	int oldsize, limit;
	struct node_buffer *nb;
	struct node *new = new_node(L);
	new->size = sz;
	new->buff = data;
	nb = (struct node_buffer *)luaL_checkudata(L, NB, "nodebuffer");
	assert(nb->sid == sid);
	limit = nb->limit;
	oldsize = nb->size;
	append_node(nb, new);
	if (oldsize < limit && nb->size >= limit)
		silly_socket_readctrl(nb->sid, 0);
{'fraction_non_alphanumeric': 0.17634880038972112, 'fraction_numerical': 0.013153087321885276, 'mean_word_length': 3.8305882352941176, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '944258', 'n_tokens_mistral': 3701, 'n_tokens_neox': 3548, 'n_words': 936}