<commit_before>/*
 *  duk_heap allocation and freeing.
 */

#include "duk_internal.h"

/* constants for built-in string data depacking */
#define DUK__BITPACK_LETTER_LIMIT  26
#define DUK__BITPACK_UNDERSCORE    26
#define DUK__BITPACK_FF            27
#define DUK__BITPACK_SWITCH1       29
#define DUK__BITPACK_SWITCH        30
#define DUK__BITPACK_SEVENBIT      31

/*
 *  Free a heap object.
 *
 *  Free heap object and its internal (non-heap) pointers.  Assumes that
 *  caller has removed the object from heap allocated list or the string
 *  intern table, and any weak references (which strings may have) have
 *  been already dealt with.
 */

static void duk__free_hobject_inner(duk_heap *heap, duk_hobject *h) {
	DUK_ASSERT(heap != NULL);
	DUK_ASSERT(h != NULL);

	DUK_FREE(heap, h->p);

	if (DUK_HOBJECT_IS_COMPILEDFUNCTION(h)) {
		duk_hcompiledfunction *f = (duk_hcompiledfunction *) h;
		DUK_UNREF(f);
		/* Currently nothing to free; 'data' is a heap object */
	} else if (DUK_HOBJECT_IS_NATIVEFUNCTION(h)) {
		duk_hnativefunction *f = (duk_hnativefunction *) h;
		DUK_UNREF(f);
		/* Currently nothing to free */
	} else if (DUK_HOBJECT_IS_THREAD(h)) {
		duk_hthread *t = (duk_hthread *) h;
		DUK_FREE(heap, t->valstack);
		DUK_FREE(heap, t->callstack);
		DUK_FREE(heap, t->catchstack);
		/* Don't free h->resumer because it exists in the heap.
		 * Callstack entries also contain function pointers which
		 * are not freed for the same reason.
		 */

		/* XXX: with 'caller' property the callstack would need
		 * to be unwound to update the 'caller' properties of
		 * functions in the callstack.
		 */
	}
}

static void duk__free_hbuffer_inner(duk_heap *heap, duk_hbuffer *h) {
	DUK_ASSERT(heap != NULL);
	DUK_ASSERT(h != NULL);

	if (DUK_HBUFFER_HAS_DYNAMIC(h)) {
		duk_hbuffer_dynamic *g = (duk_hbuffer_dynamic *) h;
		DUK_DDD(DUK_DDDPRINT("free dynamic buffer %p", (void *) g->curr_alloc));
		DUK_FREE(heap, g->curr_alloc);
	}
}

void duk_heap_free_heaphdr_raw(duk_heap *heap, duk_heaphdr *hdr) {
	DUK_ASSERT(heap);
	DUK_ASSERT(hdr);

	DUK_DDD(DUK_DDDPRINT("free heaphdr %p, htype %ld", (void *) hdr, (long) DUK_HEAPHDR_GET_TYPE(hdr)));

	switch ((int) DUK_HEAPHDR_GET_TYPE(hdr)) {
	case DUK_HTYPE_STRING:
		/* no inner refs to free */
		break;
	case DUK_HTYPE_OBJECT:
		duk__free_hobject_inner(heap, (duk_hobject *) hdr);
		break;
	case DUK_HTYPE_BUFFER:
		duk__free_hbuffer_inner(heap, (duk_hbuffer *) hdr);
		break;
	default:
		DUK_UNREACHABLE();
	}

	DUK_FREE(heap, hdr);
}

/*
 *  Free the heap.
 *
 *  Frees heap-related non-heap-tracked allocations such as the
 *  string intern table; then frees the heap allocated objects;
 *  and finally frees the heap structure itself.  Reference counts
 *  and GC markers are ignored (and not updated) in this process,
 *  and finalizers won't be called.
 *
 *  The heap pointer and heap object pointers must not be used
 *  after this call.
 */

static void duk__free_allocated(duk_heap *heap) {
	duk_heaphdr *curr;
	duk_heaphdr *next;

	curr = heap->heap_allocated;
	while (curr) {
		/* We don't log or warn about freeing zero refcount objects
		 * because they may happen with finalizer processing.
		 */

		DUK_DDD(DUK_DDDPRINT("FINALFREE (allocated): %!iO",
		                     (duk_heaphdr *) curr));
		next = DUK_HEAPHDR_GET_NEXT(curr);
		duk_heap_free_heaphdr_raw(heap, curr);
		curr = next;
	}
}

#ifdef DUK_USE_REFERENCE_COUNTING
static void duk__free_refzero_list(duk_heap *heap) {
	duk_heaphdr *curr;
	duk_heaphdr *next;

	curr = heap->refzero_list;
	while (curr) {
		DUK_DDD(DUK_DDDPRINT("FINALFREE (refzero_list): %!iO",
		                     (duk_heaphdr *) curr));
		next = DUK_HEAPHDR_GET_NEXT(curr);
		duk_heap_free_heaphdr_raw(heap, curr);
		curr = next;
	}
}
#endif

#ifdef DUK_USE_MARK_AND_SWEEP
static void duk__free_markandsweep_finalize_list(duk_heap *heap) {
	duk_heaphdr *curr;
	duk_heaphdr *next;

	curr = heap->finalize_list;
	while (curr) {
		DUK_DDD(DUK_DDDPRINT("FINALFREE (finalize_list): %!iO",
		                     (duk_heaphdr *) curr));
		next = DUK_HEAPHDR_GET_NEXT(curr);
		duk_heap_free_heaphdr_raw(heap, curr);
		curr = next;
	}
}
#endif

static void duk__free_stringtable(duk_heap *heap) {
	duk_uint_fast32_t i;

	/* strings are only tracked by stringtable */
	if (heap->st) {
		for (i = 0; i < (duk_uint_fast32_t) heap->st_size; i++) {
			duk_hstring *e = heap->st[i];
			if (e == DUK_STRTAB_DELETED_MARKER(heap)) {
				continue;
			}

			/* strings have no inner allocations so free directly */
			DUK_DDD(DUK_DDDPRINT("FINALFREE (string): %!iO",
			                     (duk_heaphdr *) e));
			DUK_FREE(heap, e);
#if 0  /* not strictly necessary */
			heap->st[i] = NULL;
#endif
		}
		DUK_FREE(heap, heap->st);
#if 0  /* not strictly necessary */
		heap->st = NULL;
#endif
	}
}

static void duk__free_run_finalizers(duk_heap *heap) {
	duk_heaphdr *curr;
#ifdef DUK_USE_DEBUG
	duk_size_t count_obj = 0;
#endif

	DUK_ASSERT(heap != NULL);
	DUK_ASSERT(heap->heap_thread != NULL);
#ifdef DUK_USE_REFERENCE_COUNTING
	DUK_ASSERT(heap->refzero_list == NULL);  /* refzero not running -> must be empty */
#endif
#ifdef DUK_USE_MARK_AND_SWEEP
	DUK_ASSERT(heap->finalize_list == NULL);  /* mark-and-sweep not running -> must be empty */
#endif

	/* FIXME: here again finalizer thread is the heap_thread which needs
	 * to be coordinated with finalizer thread fixes.
	 */

	curr = heap->heap_allocated;
	while (curr) {
		if (DUK_HEAPHDR_GET_TYPE(curr) == DUK_HTYPE_OBJECT) {
			/* Only objects in heap_allocated may have finalizers. */
			DUK_ASSERT(heap->heap_thread != NULL);
			DUK_ASSERT(curr != NULL);
			duk_hobject_run_finalizer(heap->heap_thread, (duk_hobject *) curr);
#ifdef DUK_USE_DEBUG
			count_obj++;
#endif
		}
		curr = DUK_HEAPHDR_GET_NEXT(curr);
	}

	/* Note: count includes all objects, not only those with an actual finalizer. */
#ifdef DUK_USE_DEBUG
	DUK_D(DUK_DPRINT("checked %ld objects for finalizers before freeing heap", (long) count_obj));
#endif
}

void duk_heap_free(duk_heap *heap) {
	DUK_D(DUK_DPRINT("free heap: %p", (void *) heap));

	/* Execute finalizers before freeing the heap, even for reachable
	 * objects, and regardless of whether or not mark-and-sweep is
	 * enabled.  This gives finalizers the chance to free any native
	 * resources like file handles, allocations made outside Duktape,
	 * etc.
	 *
	 * FIXME: this perhaps requires an execution time limit.
	 */
	DUK_D(DUK_DPRINT("execute finalizers before freeing heap"));
#ifdef DUK_USE_MARK_AND_SWEEP
	/* run mark-and-sweep a few times just in case (unreachable
	 * object finalizers run already here)
	 */
	duk_heap_mark_and_sweep(heap, 0);
	duk_heap_mark_and_sweep(heap, 0);
#endif
	duk__free_run_finalizers(heap);

	/* Note: heap->heap_thread, heap->curr_thread, heap->heap_object,
	 * and heap->log_buffer are on the heap allocated list.
	 */

	DUK_D(DUK_DPRINT("freeing heap objects of heap: %p", (void *) heap));
	duk__free_allocated(heap);

#ifdef DUK_USE_REFERENCE_COUNTING
	DUK_D(DUK_DPRINT("freeing refzero list of heap: %p", (void *) heap));
	duk__free_refzero_list(heap);
#endif

#ifdef DUK_USE_MARK_AND_SWEEP
	DUK_D(DUK_DPRINT("freeing mark-and-sweep finalize list of heap: %p", (void *) heap));
	duk__free_markandsweep_finalize_list(heap);
#endif

	DUK_D(DUK_DPRINT("freeing string table of heap: %p", (void *) heap));
	duk__free_stringtable(heap);

	DUK_D(DUK_DPRINT("freeing heap structure: %p", (void *) heap));
	heap->free_func(heap->alloc_udata, heap);
}

/*
 *  Allocate a heap.
 *
 *  String table is initialized with built-in strings from genstrings.py.
 */

/* intern built-in strings from precooked data (genstrings.py) */
static int duk__init_heap_strings(duk_heap *heap) {
	duk_bitdecoder_ctx bd_ctx;
	duk_bitdecoder_ctx *bd = &bd_ctx;  /* convenience */
	duk_small_uint_t i, j;

	DUK_MEMZERO(&bd_ctx, sizeof(bd_ctx));
	bd->data = (const duk_uint8_t *) duk_strings_data;
	bd->length = (duk_size_t) DUK_STRDATA_DATA_LENGTH;

	for (i = 0; i < DUK_HEAP_NUM_STRINGS; i++) {
		duk_uint8_t tmp[DUK_STRDATA_MAX_STRLEN];
		duk_hstring *h;
		duk_small_uint_t len;
		duk_small_uint_t mode;
		duk_small_uint_t t;

		len = duk_bd_decode(bd, 5);
		mode = 32;		/* 0 = uppercase, 32 = lowercase (= 'a' - 'A') */
		for (j = 0; j < len; j++) {
			t = duk_bd_decode(bd, 5);
			if (t < DUK__BITPACK_LETTER_LIMIT) {
				t = t + DUK_ASC_UC_A + mode;
			} else if (t == DUK__BITPACK_UNDERSCORE) {
				t = DUK_ASC_UNDERSCORE;
			} else if (t == DUK__BITPACK_FF) {
				/* Internal keys are prefixed with 0xFF in the stringtable
				 * (which makes them invalid UTF-8 on purpose).
				 */
				t = 0xff;
			} else if (t == DUK__BITPACK_SWITCH1) {
				t = duk_bd_decode(bd, 5);
				DUK_ASSERT_DISABLE(t >= 0);  /* unsigned */
				DUK_ASSERT(t <= 25);
				t = t + DUK_ASC_UC_A + (mode ^ 32);
			} else if (t == DUK__BITPACK_SWITCH) {
				mode = mode ^ 32;
				t = duk_bd_decode(bd, 5);
				DUK_ASSERT_DISABLE(t >= 0);
				DUK_ASSERT(t <= 25);
				t = t + DUK_ASC_UC_A + mode;
			} else if (t == DUK__BITPACK_SEVENBIT) {
				t = duk_bd_decode(bd, 7);
			}
			tmp[j] = (duk_uint8_t) t;
		}

		DUK_DDD(DUK_DDDPRINT("intern built-in string %ld", (long) i));
		h = duk_heap_string_intern(heap, tmp, len);
		if (!h) {
			goto error;
		}

		/* special flags */

		if (len > 0 && tmp[0] == (duk_uint8_t) 0xff) {
			DUK_HSTRING_SET_INTERNAL(h);
		}
		if (i == DUK_STRIDX_EVAL || i == DUK_STRIDX_LC_ARGUMENTS) {
			DUK_HSTRING_SET_EVAL_OR_ARGUMENTS(h);
		}
		if (i >= DUK_STRIDX_START_RESERVED && i < DUK_STRIDX_END_RESERVED) {
			DUK_HSTRING_SET_RESERVED_WORD(h);
			if (i >= DUK_STRIDX_START_STRICT_RESERVED) {
				DUK_HSTRING_SET_STRICT_RESERVED_WORD(h);
			}
		}

		DUK_DDD(DUK_DDDPRINT("interned: %!O", (duk_heaphdr *) h));

		/* XXX: The incref macro takes a thread pointer but doesn't
		 * use it right now.
		 */
		DUK_HSTRING_INCREF(_never_referenced_, h);

		heap->strs[i] = h;
	}

	return 1;

 error:
	return 0;
}

static int duk__init_heap_thread(duk_heap *heap) {
	duk_hthread *thr;
	
	DUK_DD(DUK_DDPRINT("heap init: alloc heap thread"));
	thr = duk_hthread_alloc(heap,
	                        DUK_HOBJECT_FLAG_EXTENSIBLE |
	                        DUK_HOBJECT_FLAG_THREAD |
	                        DUK_HOBJECT_CLASS_AS_FLAGS(DUK_HOBJECT_CLASS_THREAD));
	if (!thr) {
		DUK_D(DUK_DPRINT("failed to alloc heap_thread"));
		return 0;
	}
	thr->state = DUK_HTHREAD_STATE_INACTIVE;
	thr->strs = heap->strs;

	heap->heap_thread = thr;
	DUK_HTHREAD_INCREF(thr, thr);  /* Note: first argument not really used */

	/* 'thr' is now reachable */

	if (!duk_hthread_init_stacks(heap, thr)) {
		return 0;
	}

	/* FIXME: this may now fail, and is not handled correctly */
	duk_hthread_create_builtin_objects(thr);

	/* default prototype (Note: 'thr' must be reachable) */
	DUK_HOBJECT_SET_PROTOTYPE_UPDREF(thr, (duk_hobject *) thr, thr->builtins[DUK_BIDX_THREAD_PROTOTYPE]);

	return 1;
}

#ifdef DUK_USE_DEBUG
#define DUK__DUMPSZ(t)  do { \
		DUK_D(DUK_DPRINT("" #t "=%ld", (long) sizeof(t))); \
	} while (0)

/* These is not 100% because format would need to be non-portable "long long". */
#define DUK__DUMPLM_SIGNED_RAW(t,a,b)  do { \
		DUK_D(DUK_DPRINT(t "=[%ld,%ld]", (long) (a), (long) (b))); \
	} while(0)
#define DUK__DUMPLM_UNSIGNED_RAW(t,a,b)  do { \
		DUK_D(DUK_DPRINT(t "=[%lu,%lu]", (unsigned long) (a), (unsigned long) (b))); \
	} while(0)
#define DUK__DUMPLM_SIGNED(t)  do { \
		DUK__DUMPLM_SIGNED_RAW("DUK_" #t "_{MIN,MAX}", DUK_##t##_MIN, DUK_##t##_MAX); \
	} while(0)
#define DUK__DUMPLM_UNSIGNED(t)  do { \
		DUK__DUMPLM_UNSIGNED_RAW("DUK_" #t "_{MIN,MAX}", DUK_##t##_MIN, DUK_##t##_MAX); \
	} while(0)<commit_msg>Improve int type limit debug printing<commit_after>/*
 *  duk_heap allocation and freeing.
 */

#include "duk_internal.h"

/* constants for built-in string data depacking */
#define DUK__BITPACK_LETTER_LIMIT  26
#define DUK__BITPACK_UNDERSCORE    26
#define DUK__BITPACK_FF            27
#define DUK__BITPACK_SWITCH1       29
#define DUK__BITPACK_SWITCH        30
#define DUK__BITPACK_SEVENBIT      31

/*
 *  Free a heap object.
 *
 *  Free heap object and its internal (non-heap) pointers.  Assumes that
 *  caller has removed the object from heap allocated list or the string
 *  intern table, and any weak references (which strings may have) have
 *  been already dealt with.
 */

static void duk__free_hobject_inner(duk_heap *heap, duk_hobject *h) {
	DUK_ASSERT(heap != NULL);
	DUK_ASSERT(h != NULL);

	DUK_FREE(heap, h->p);

	if (DUK_HOBJECT_IS_COMPILEDFUNCTION(h)) {
		duk_hcompiledfunction *f = (duk_hcompiledfunction *) h;
		DUK_UNREF(f);
		/* Currently nothing to free; 'data' is a heap object */
	} else if (DUK_HOBJECT_IS_NATIVEFUNCTION(h)) {
		duk_hnativefunction *f = (duk_hnativefunction *) h;
		DUK_UNREF(f);
		/* Currently nothing to free */
	} else if (DUK_HOBJECT_IS_THREAD(h)) {
		duk_hthread *t = (duk_hthread *) h;
		DUK_FREE(heap, t->valstack);
		DUK_FREE(heap, t->callstack);
		DUK_FREE(heap, t->catchstack);
		/* Don't free h->resumer because it exists in the heap.
		 * Callstack entries also contain function pointers which
		 * are not freed for the same reason.
		 */

		/* XXX: with 'caller' property the callstack would need
		 * to be unwound to update the 'caller' properties of
		 * functions in the callstack.
		 */
	}
}

static void duk__free_hbuffer_inner(duk_heap *heap, duk_hbuffer *h) {
	DUK_ASSERT(heap != NULL);
	DUK_ASSERT(h != NULL);

	if (DUK_HBUFFER_HAS_DYNAMIC(h)) {
		duk_hbuffer_dynamic *g = (duk_hbuffer_dynamic *) h;
		DUK_DDD(DUK_DDDPRINT("free dynamic buffer %p", (void *) g->curr_alloc));
		DUK_FREE(heap, g->curr_alloc);
	}
}

void duk_heap_free_heaphdr_raw(duk_heap *heap, duk_heaphdr *hdr) {
	DUK_ASSERT(heap);
	DUK_ASSERT(hdr);

	DUK_DDD(DUK_DDDPRINT("free heaphdr %p, htype %ld", (void *) hdr, (long) DUK_HEAPHDR_GET_TYPE(hdr)));

	switch ((int) DUK_HEAPHDR_GET_TYPE(hdr)) {
	case DUK_HTYPE_STRING:
		/* no inner refs to free */
		break;
	case DUK_HTYPE_OBJECT:
		duk__free_hobject_inner(heap, (duk_hobject *) hdr);
		break;
	case DUK_HTYPE_BUFFER:
		duk__free_hbuffer_inner(heap, (duk_hbuffer *) hdr);
		break;
	default:
		DUK_UNREACHABLE();
	}

	DUK_FREE(heap, hdr);
}

/*
 *  Free the heap.
 *
 *  Frees heap-related non-heap-tracked allocations such as the
 *  string intern table; then frees the heap allocated objects;
 *  and finally frees the heap structure itself.  Reference counts
 *  and GC markers are ignored (and not updated) in this process,
 *  and finalizers won't be called.
 *
 *  The heap pointer and heap object pointers must not be used
 *  after this call.
 */

static void duk__free_allocated(duk_heap *heap) {
	duk_heaphdr *curr;
	duk_heaphdr *next;

	curr = heap->heap_allocated;
	while (curr) {
		/* We don't log or warn about freeing zero refcount objects
		 * because they may happen with finalizer processing.
		 */

		DUK_DDD(DUK_DDDPRINT("FINALFREE (allocated): %!iO",
		                     (duk_heaphdr *) curr));
		next = DUK_HEAPHDR_GET_NEXT(curr);
		duk_heap_free_heaphdr_raw(heap, curr);
		curr = next;
	}
}

#ifdef DUK_USE_REFERENCE_COUNTING
static void duk__free_refzero_list(duk_heap *heap) {
	duk_heaphdr *curr;
	duk_heaphdr *next;

	curr = heap->refzero_list;
	while (curr) {
		DUK_DDD(DUK_DDDPRINT("FINALFREE (refzero_list): %!iO",
		                     (duk_heaphdr *) curr));
		next = DUK_HEAPHDR_GET_NEXT(curr);
		duk_heap_free_heaphdr_raw(heap, curr);
		curr = next;
	}
}
#endif

#ifdef DUK_USE_MARK_AND_SWEEP
static void duk__free_markandsweep_finalize_list(duk_heap *heap) {
	duk_heaphdr *curr;
	duk_heaphdr *next;

	curr = heap->finalize_list;
	while (curr) {
		DUK_DDD(DUK_DDDPRINT("FINALFREE (finalize_list): %!iO",
		                     (duk_heaphdr *) curr));
		next = DUK_HEAPHDR_GET_NEXT(curr);
		duk_heap_free_heaphdr_raw(heap, curr);
		curr = next;
	}
}
#endif

static void duk__free_stringtable(duk_heap *heap) {
	duk_uint_fast32_t i;

	/* strings are only tracked by stringtable */
	if (heap->st) {
		for (i = 0; i < (duk_uint_fast32_t) heap->st_size; i++) {
			duk_hstring *e = heap->st[i];
			if (e == DUK_STRTAB_DELETED_MARKER(heap)) {
				continue;
			}

			/* strings have no inner allocations so free directly */
			DUK_DDD(DUK_DDDPRINT("FINALFREE (string): %!iO",
			                     (duk_heaphdr *) e));
			DUK_FREE(heap, e);
#if 0  /* not strictly necessary */
			heap->st[i] = NULL;
#endif
		}
		DUK_FREE(heap, heap->st);
#if 0  /* not strictly necessary */
		heap->st = NULL;
#endif
	}
}

static void duk__free_run_finalizers(duk_heap *heap) {
	duk_heaphdr *curr;
#ifdef DUK_USE_DEBUG
	duk_size_t count_obj = 0;
#endif

	DUK_ASSERT(heap != NULL);
	DUK_ASSERT(heap->heap_thread != NULL);
#ifdef DUK_USE_REFERENCE_COUNTING
	DUK_ASSERT(heap->refzero_list == NULL);  /* refzero not running -> must be empty */
#endif
#ifdef DUK_USE_MARK_AND_SWEEP
	DUK_ASSERT(heap->finalize_list == NULL);  /* mark-and-sweep not running -> must be empty */
#endif

	/* FIXME: here again finalizer thread is the heap_thread which needs
	 * to be coordinated with finalizer thread fixes.
	 */

	curr = heap->heap_allocated;
	while (curr) {
		if (DUK_HEAPHDR_GET_TYPE(curr) == DUK_HTYPE_OBJECT) {
			/* Only objects in heap_allocated may have finalizers. */
			DUK_ASSERT(heap->heap_thread != NULL);
			DUK_ASSERT(curr != NULL);
			duk_hobject_run_finalizer(heap->heap_thread, (duk_hobject *) curr);
#ifdef DUK_USE_DEBUG
			count_obj++;
#endif
		}
		curr = DUK_HEAPHDR_GET_NEXT(curr);
	}

	/* Note: count includes all objects, not only those with an actual finalizer. */
#ifdef DUK_USE_DEBUG
	DUK_D(DUK_DPRINT("checked %ld objects for finalizers before freeing heap", (long) count_obj));
#endif
}

void duk_heap_free(duk_heap *heap) {
	DUK_D(DUK_DPRINT("free heap: %p", (void *) heap));

	/* Execute finalizers before freeing the heap, even for reachable
	 * objects, and regardless of whether or not mark-and-sweep is
	 * enabled.  This gives finalizers the chance to free any native
	 * resources like file handles, allocations made outside Duktape,
	 * etc.
	 *
	 * FIXME: this perhaps requires an execution time limit.
	 */
	DUK_D(DUK_DPRINT("execute finalizers before freeing heap"));
#ifdef DUK_USE_MARK_AND_SWEEP
	/* run mark-and-sweep a few times just in case (unreachable
	 * object finalizers run already here)
	 */
	duk_heap_mark_and_sweep(heap, 0);
	duk_heap_mark_and_sweep(heap, 0);
#endif
	duk__free_run_finalizers(heap);

	/* Note: heap->heap_thread, heap->curr_thread, heap->heap_object,
	 * and heap->log_buffer are on the heap allocated list.
	 */

	DUK_D(DUK_DPRINT("freeing heap objects of heap: %p", (void *) heap));
	duk__free_allocated(heap);

#ifdef DUK_USE_REFERENCE_COUNTING
	DUK_D(DUK_DPRINT("freeing refzero list of heap: %p", (void *) heap));
	duk__free_refzero_list(heap);
#endif

#ifdef DUK_USE_MARK_AND_SWEEP
	DUK_D(DUK_DPRINT("freeing mark-and-sweep finalize list of heap: %p", (void *) heap));
	duk__free_markandsweep_finalize_list(heap);
#endif

	DUK_D(DUK_DPRINT("freeing string table of heap: %p", (void *) heap));
	duk__free_stringtable(heap);

	DUK_D(DUK_DPRINT("freeing heap structure: %p", (void *) heap));
	heap->free_func(heap->alloc_udata, heap);
}

/*
 *  Allocate a heap.
 *
 *  String table is initialized with built-in strings from genstrings.py.
 */

/* intern built-in strings from precooked data (genstrings.py) */
static int duk__init_heap_strings(duk_heap *heap) {
	duk_bitdecoder_ctx bd_ctx;
	duk_bitdecoder_ctx *bd = &bd_ctx;  /* convenience */
	duk_small_uint_t i, j;

	DUK_MEMZERO(&bd_ctx, sizeof(bd_ctx));
	bd->data = (const duk_uint8_t *) duk_strings_data;
	bd->length = (duk_size_t) DUK_STRDATA_DATA_LENGTH;

	for (i = 0; i < DUK_HEAP_NUM_STRINGS; i++) {
		duk_uint8_t tmp[DUK_STRDATA_MAX_STRLEN];
		duk_hstring *h;
		duk_small_uint_t len;
		duk_small_uint_t mode;
		duk_small_uint_t t;

		len = duk_bd_decode(bd, 5);
		mode = 32;		/* 0 = uppercase, 32 = lowercase (= 'a' - 'A') */
		for (j = 0; j < len; j++) {
			t = duk_bd_decode(bd, 5);
			if (t < DUK__BITPACK_LETTER_LIMIT) {
				t = t + DUK_ASC_UC_A + mode;
			} else if (t == DUK__BITPACK_UNDERSCORE) {
				t = DUK_ASC_UNDERSCORE;
			} else if (t == DUK__BITPACK_FF) {
				/* Internal keys are prefixed with 0xFF in the stringtable
				 * (which makes them invalid UTF-8 on purpose).
				 */
				t = 0xff;
			} else if (t == DUK__BITPACK_SWITCH1) {
				t = duk_bd_decode(bd, 5);
				DUK_ASSERT_DISABLE(t >= 0);  /* unsigned */
				DUK_ASSERT(t <= 25);
				t = t + DUK_ASC_UC_A + (mode ^ 32);
			} else if (t == DUK__BITPACK_SWITCH) {
				mode = mode ^ 32;
				t = duk_bd_decode(bd, 5);
				DUK_ASSERT_DISABLE(t >= 0);
				DUK_ASSERT(t <= 25);
				t = t + DUK_ASC_UC_A + mode;
			} else if (t == DUK__BITPACK_SEVENBIT) {
				t = duk_bd_decode(bd, 7);
			}
			tmp[j] = (duk_uint8_t) t;
		}

		DUK_DDD(DUK_DDDPRINT("intern built-in string %ld", (long) i));
		h = duk_heap_string_intern(heap, tmp, len);
		if (!h) {
			goto error;
		}

		/* special flags */

		if (len > 0 && tmp[0] == (duk_uint8_t) 0xff) {
			DUK_HSTRING_SET_INTERNAL(h);
		}
		if (i == DUK_STRIDX_EVAL || i == DUK_STRIDX_LC_ARGUMENTS) {
			DUK_HSTRING_SET_EVAL_OR_ARGUMENTS(h);
		}
		if (i >= DUK_STRIDX_START_RESERVED && i < DUK_STRIDX_END_RESERVED) {
			DUK_HSTRING_SET_RESERVED_WORD(h);
			if (i >= DUK_STRIDX_START_STRICT_RESERVED) {
				DUK_HSTRING_SET_STRICT_RESERVED_WORD(h);
			}
		}

		DUK_DDD(DUK_DDDPRINT("interned: %!O", (duk_heaphdr *) h));

		/* XXX: The incref macro takes a thread pointer but doesn't
		 * use it right now.
		 */
		DUK_HSTRING_INCREF(_never_referenced_, h);

		heap->strs[i] = h;
	}

	return 1;

 error:
	return 0;
}

static int duk__init_heap_thread(duk_heap *heap) {
	duk_hthread *thr;
	
	DUK_DD(DUK_DDPRINT("heap init: alloc heap thread"));
	thr = duk_hthread_alloc(heap,
	                        DUK_HOBJECT_FLAG_EXTENSIBLE |
	                        DUK_HOBJECT_FLAG_THREAD |
	                        DUK_HOBJECT_CLASS_AS_FLAGS(DUK_HOBJECT_CLASS_THREAD));
	if (!thr) {
		DUK_D(DUK_DPRINT("failed to alloc heap_thread"));
		return 0;
	}
	thr->state = DUK_HTHREAD_STATE_INACTIVE;
	thr->strs = heap->strs;

	heap->heap_thread = thr;
	DUK_HTHREAD_INCREF(thr, thr);  /* Note: first argument not really used */

	/* 'thr' is now reachable */

	if (!duk_hthread_init_stacks(heap, thr)) {
		return 0;
	}

	/* FIXME: this may now fail, and is not handled correctly */
	duk_hthread_create_builtin_objects(thr);

	/* default prototype (Note: 'thr' must be reachable) */
	DUK_HOBJECT_SET_PROTOTYPE_UPDREF(thr, (duk_hobject *) thr, thr->builtins[DUK_BIDX_THREAD_PROTOTYPE]);

	return 1;
}

#ifdef DUK_USE_DEBUG
#define DUK__DUMPSZ(t)  do { \
		DUK_D(DUK_DPRINT("" #t "=%ld", (long) sizeof(t))); \
	} while (0)

/* These is not 100% because format would need to be non-portable "long long".
 * Also print out as doubles to catch cases where the "long" type is not wide
 * enough; the limits will then not be printed accurately but the magnitude
 * will be correct.
 */
#define DUK__DUMPLM_SIGNED_RAW(t,a,b)  do { \
		DUK_D(DUK_DPRINT(t "=[%ld,%ld]=[%lf,%lf]", \
		                 (long) (a), (long) (b), \
		                 (double) (a), (double) (b))); \
	} while(0)
#define DUK__DUMPLM_UNSIGNED_RAW(t,a,b)  do { \
		DUK_D(DUK_DPRINT(t "=[%lu,%lu]=[%lf,%lf]", \
		                 (unsigned long) (a), (unsigned long) (b), \
		                 (double) (a), (double) (b))); \
	} while(0)
#define DUK__DUMPLM_SIGNED(t)  do { \
		DUK__DUMPLM_SIGNED_RAW("DUK_" #t "_{MIN,MAX}", DUK_##t##_MIN, DUK_##t##_MAX); \
	} while(0)
#define DUK__DUMPLM_UNSIGNED(t)  do { \
		DUK__DUMPLM_UNSIGNED_RAW("DUK_" #t "_{MIN,MAX}", DUK_##t##_MIN, DUK_##t##_MAX); \
	} while(0)
{'fraction_non_alphanumeric': 0.12548205280332245, 'fraction_numerical': 0.005848201042505404, 'mean_word_length': 3.9430247172182655, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1030608', 'n_tokens_mistral': 10120, 'n_tokens_neox': 9390, 'n_words': 2154}