<commit_before>#include "StdLib.h"

#include "nullc.h"
#include "nullc_debug.h"
#include <string.h>
#include <time.h>

#include "stdafx.h"
#include "Pool.h"
#include "Tree.h"

#include "Executor_Common.h"
#include "includes/typeinfo.h"

// memory structure				   |base->
// small object storage:			marker, data...
// big object storage:		size,	marker, data...
// small array storage:				marker, count, data...
// big array storage:		size,	marker, count, data...

namespace NULLC
{
	static Linker	*linker = NULL;
	FastVector<NULLCRef>	finalizeList;

	static uintptr_t OBJECT_VISIBLE		= 1 << 0;
	static uintptr_t OBJECT_FREED		= 1 << 1;
	static uintptr_t OBJECT_FINALIZABLE	= 1 << 2;
	static uintptr_t OBJECT_FINALIZED	= 1 << 3;
	static uintptr_t OBJECT_ARRAY		= 1 << 4;
	static uintptr_t OBJECT_MASK		= OBJECT_VISIBLE | OBJECT_FREED;

	void FinalizeObject(markerType& marker, char* base)
	{
		if(marker & NULLC::OBJECT_ARRAY)
		{
			ExternTypeInfo &typeInfo = NULLC::linker->exTypes[(unsigned)marker >> 8];

			unsigned arrayPadding = typeInfo.defaultAlign > 4 ? typeInfo.defaultAlign : 4;

			unsigned count = *(unsigned*)(base + sizeof(markerType) + arrayPadding - 4);
			NULLCRef r = { (unsigned)marker >> 8, base + sizeof(markerType) + arrayPadding }; // skip over marker and array size

			for(unsigned i = 0; i < count; i++)
			{
				NULLC::finalizeList.push_back(r);
				r.ptr += typeInfo.size;
			}
		}else{
			NULLCRef r = { (unsigned)marker >> 8, base + sizeof(markerType) }; // skip over marker
			NULLC::finalizeList.push_back(r);
		}
		marker |= NULLC::OBJECT_FINALIZED;
	}
}

template<int elemSize>
union SmallBlock
{
	char			data[elemSize];
	markerType		marker;
	SmallBlock		*next;
};

#pragma pack(push, 1)
template<int elemSize, int countInBlock>
struct LargeBlock
{
	typedef SmallBlock<elemSize> Block;

	// Padding is used to break the 16 byte alignment of pages in a way that after a marker offset is added to the block, the object pointer will be correctly aligned
	char padding[16 - sizeof(markerType)];
	Block		page[countInBlock];

	LargeBlock	*next;
};
#pragma pack(pop)

template<int elemSize, int countInBlock>
class ObjectBlockPool
{
	typedef SmallBlock<elemSize> MySmallBlock;
	typedef LargeBlock<elemSize, countInBlock> MyLargeBlock;
public:
	ObjectBlockPool()
	{
		freeBlocks = &lastBlock;
		activePages = NULL;
		lastNum = countInBlock;
	}
	~ObjectBlockPool()
	{
		if(!activePages)
			return;
		do
		{
			MyLargeBlock* following = activePages->next;
			NULLC::alignedDealloc(activePages);
			activePages = following;
		}while(activePages != NULL);
		freeBlocks = &lastBlock;
		activePages = NULL;
		lastNum = countInBlock;
		sortedPages.reset();
	}

	void* Alloc()
	{
		MySmallBlock*	result;
		if(freeBlocks && freeBlocks != &lastBlock)
		{
			result = freeBlocks;
			freeBlocks = (MySmallBlock*)((intptr_t)freeBlocks->next & ~NULLC::OBJECT_MASK);
		}else{
			if(lastNum == countInBlock)
			{
				MyLargeBlock* newPage = new(NULLC::alignedAlloc(sizeof(MyLargeBlock))) MyLargeBlock;
				newPage->next = activePages;
				activePages = newPage;
				lastNum = 0;
				sortedPages.push_back(newPage);
				int index = sortedPages.size() - 1;
				while(index > 0 && sortedPages[index] < sortedPages[index - 1])
				{<commit_msg>Fixed crash when GC finds an address to an object that was not allocated yet<commit_after>#include "StdLib.h"

#include "nullc.h"
#include "nullc_debug.h"
#include <string.h>
#include <time.h>

#include "stdafx.h"
#include "Pool.h"
#include "Tree.h"

#include "Executor_Common.h"
#include "includes/typeinfo.h"

// memory structure				   |base->
// small object storage:			marker, data...
// big object storage:		size,	marker, data...
// small array storage:				marker, count, data...
// big array storage:		size,	marker, count, data...

namespace NULLC
{
	static Linker	*linker = NULL;
	FastVector<NULLCRef>	finalizeList;

	static uintptr_t OBJECT_VISIBLE		= 1 << 0;
	static uintptr_t OBJECT_FREED		= 1 << 1;
	static uintptr_t OBJECT_FINALIZABLE	= 1 << 2;
	static uintptr_t OBJECT_FINALIZED	= 1 << 3;
	static uintptr_t OBJECT_ARRAY		= 1 << 4;
	static uintptr_t OBJECT_MASK		= OBJECT_VISIBLE | OBJECT_FREED;

	void FinalizeObject(markerType& marker, char* base)
	{
		if(marker & NULLC::OBJECT_ARRAY)
		{
			ExternTypeInfo &typeInfo = NULLC::linker->exTypes[(unsigned)marker >> 8];

			unsigned arrayPadding = typeInfo.defaultAlign > 4 ? typeInfo.defaultAlign : 4;

			unsigned count = *(unsigned*)(base + sizeof(markerType) + arrayPadding - 4);
			NULLCRef r = { (unsigned)marker >> 8, base + sizeof(markerType) + arrayPadding }; // skip over marker and array size

			for(unsigned i = 0; i < count; i++)
			{
				NULLC::finalizeList.push_back(r);
				r.ptr += typeInfo.size;
			}
		}else{
			NULLCRef r = { (unsigned)marker >> 8, base + sizeof(markerType) }; // skip over marker
			NULLC::finalizeList.push_back(r);
		}
		marker |= NULLC::OBJECT_FINALIZED;
	}
}

template<int elemSize>
union SmallBlock
{
	char			data[elemSize];
	markerType		marker;
	SmallBlock		*next;
};

#pragma pack(push, 1)
template<int elemSize, int countInBlock>
struct LargeBlock
{
	typedef SmallBlock<elemSize> Block;

	// Padding is used to break the 16 byte alignment of pages in a way that after a marker offset is added to the block, the object pointer will be correctly aligned
	char padding[16 - sizeof(markerType)];
	Block		page[countInBlock];

	LargeBlock	*next;
};
#pragma pack(pop)

template<int elemSize, int countInBlock>
class ObjectBlockPool
{
	typedef SmallBlock<elemSize> MySmallBlock;
	typedef LargeBlock<elemSize, countInBlock> MyLargeBlock;
public:
	ObjectBlockPool()
	{
		freeBlocks = &lastBlock;
		activePages = NULL;
		lastNum = countInBlock;
	}
	~ObjectBlockPool()
	{
		if(!activePages)
			return;
		do
		{
			MyLargeBlock* following = activePages->next;
			NULLC::alignedDealloc(activePages);
			activePages = following;
		}while(activePages != NULL);
		freeBlocks = &lastBlock;
		activePages = NULL;
		lastNum = countInBlock;
		sortedPages.reset();
	}

	void* Alloc()
	{
		MySmallBlock*	result;
		if(freeBlocks && freeBlocks != &lastBlock)
		{
			result = freeBlocks;
			freeBlocks = (MySmallBlock*)((intptr_t)freeBlocks->next & ~NULLC::OBJECT_MASK);
		}else{
			if(lastNum == countInBlock)
			{
				MyLargeBlock* newPage = new(NULLC::alignedAlloc(sizeof(MyLargeBlock))) MyLargeBlock;
				memset(newPage, 0, sizeof(MyLargeBlock));
				newPage->next = activePages;
				activePages = newPage;
				lastNum = 0;
				sortedPages.push_back(newPage);
				int index = sortedPages.size() - 1;
				while(index > 0 && sortedPages[index] < sortedPages[index - 1])
				{
{'fraction_non_alphanumeric': 0.12331668936298987, 'fraction_numerical': 0.008019367529126948, 'mean_word_length': 4.1640625, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707051', 'n_tokens_mistral': 2550, 'n_tokens_neox': 2272, 'n_words': 577}