<commit_before>/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: t; tab-width: 4 -*- */
/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is [Open Source Virtual Machine.].
 *
 * The Initial Developer of the Original Code is
 * Adobe System Incorporated.
 * Portions created by the Initial Developer are Copyright (C) 2004-2006
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *   Adobe AS3 Team
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

#include "MMgc.h"

#ifdef AVMPLUS_SAMPLER
namespace avmplus
{
	void recordAllocationSample(const void* item, size_t size);
	void recordDeallocationSample(const void* item, size_t size);	
}
#endif

namespace MMgc
{
	GCHeap *GCHeap::instance = NULL;

	/** The native VM page size (in bytes) for the current architecture */
	const int GCHeap::kNativePageSize = GCHeap::vmPageSize();

	void GCHeap::Init(GCHeapConfig& config)
	{
		GCAssert(instance == NULL);
		instance = new GCHeap(config);
	}

	void GCHeap::Destroy()
	{
		GCAssert(instance != NULL);
		delete instance;
		instance = NULL;
	}

	GCHeap::GCHeap(GCHeapConfig& c)
		: blocks(NULL),
		  blocksLen(0),
		  numDecommitted(0),
		  numAlloc(0),
		  config(c),
#ifdef MMGC_MEMORY_PROFILER
		  profiler(NULL),
		  signal(0),
#endif
	      spyFile(stdout),
		  hooksEnabled(false),
		  blocksSpanRegions(true), // TODO: see if making this false increases heap shrinkage opportunities
		  searchForOldestBlock(!blocksSpanRegions), 
		  mergeContiguousRegions(osSupportsRegionMerging())
	{		
		lastRegion  = 0;
		blocksLen   = 0;
		numAlloc    = 0;
		numDecommitted = 0;
		blocks      = NULL;

		// Initialize free lists
		HeapBlock *block = freelists;
		for (int i=0; i<kNumFreeLists; i++) {
			block->baseAddr     = NULL;
			block->size         = 0;
			block->sizePrevious = 0;
			block->prev         = block;
			block->next         = block;
			block->committed    = true;
			block->dirty 	    = true;
			block++;
		}
		

		// Create the initial heap
		ExpandHeap((int)config.initialSize);

		fixedMalloc._Init(this);

		instance = this;

		gcs = NULL;
		gcs_count = 0;
		status = kNormal;
		
		bool enableHooks = false;
		
#ifdef MMGC_MEMORY_PROFILER
		if(!config.enableProfiler) {
			const char *env = VMPI_getenv("MMGC_PROFILE");
			if(env && VMPI_strncmp(env, "1", 1) == 0)
				config.enableProfiler = true;
		}
		enableHooks = config.enableProfiler;
#endif
		
#ifdef MMGC_MEMORY_INFO
		// always track allocs in DEBUG builds
		enableHooks = true;
#endif
		
		if(enableHooks) {
#ifdef MMGC_MEMORY_PROFILER
			profiler = new MemoryProfiler();
			VMPI_WriteOnNamedSignal("MMgc::MemoryProfiler::DumpFatties", &signal);
#endif
			hooksEnabled = true; // set only after creating profiler	
		}
	}

	GCHeap::~GCHeap()
	{
		delete [] gcs;

		fixedMalloc._Destroy();

#ifdef _DEBUG
		if(numAlloc != 0)
		{
			for (unsigned int i=0; i<blocksLen; i++) 
			{
				HeapBlock *block = &blocks[i];
				if(block->inUse() && block->baseAddr)
				{
					GCDebugMsg(false, "Block 0x%x not freed\n", block->baseAddr);
#ifdef MMGC_MEMORY_PROFILER
					if(block->allocTrace)
						PrintStackTrace(block->allocTrace);
#endif
				}
			}	
			GCAssert(false);
		}
#endif
		
#ifdef MMGC_MEMORY_PROFILER
		hooksEnabled = false;
		if(profiler)
			delete profiler;
		profiler = NULL;
#endif

		FreeAll();
	}

	void* GCHeap::Alloc(int size, bool expand/*=true*/, bool zero/*true*/)
	{
		GCAssert(size > 0);

		char *baseAddr = 0;

		// nested block to keep memset/memory commit out of critical section
		{
			MMGC_LOCK(m_spinlock);

			HeapBlock *block = AllocBlock(size, zero);

			if (!block && expand) {
				ExpandHeapLocked(size);		
				block = AllocBlock(size, zero);
			}

			if (block) 
			{			
				GCAssert(block->size == size);
			
				numAlloc += size;

				// copy baseAddr to a stack variable to fix :
				// http://flashqa.macromedia.com/bugapp/detail.asp?ID=125938
				baseAddr = block->baseAddr;
			}
		}

		if(!baseAddr && expand) {
			Abort();
		}
		
		if(baseAddr)
		{
			// Zero out the memory, if requested to do so
			// FIXME: if we know that this memory was just committed we shouldn't do this
			// maybe zero should pass deeper into the system so that on initial expansions
			// and re-commit we avoid this, should be a noise optimization over all but
			// possibly a big win for startup
			if (zero) {
				VMPI_memset(baseAddr, 0, size * kBlockSize);
			}
		}
		
		return baseAddr;
	}

	void GCHeap::Free(void *item)
	{
		MMGC_LOCK(m_spinlock);

		HeapBlock *block = AddrToBlock(item);
		if (block) {
			// Update metrics
			GCAssert(numAlloc >= (unsigned int)block->size);
			numAlloc -= block->size;
			
			FreeBlock(block);
		}
	}

	
	size_t GCHeap::Size(const void *item)
	{
		MMGC_LOCK(m_spinlock);
		HeapBlock *block = AddrToBlock(item);
		return block->size;
	}

	void GCHeap::Decommit()
	{
		// keep at least initialSize free 
		if(!config.returnMemory || GetFreeHeapSize() <= config.initialSize)
			return;
		
		size_t heapSize = GetTotalHeapSize();
		size_t freeSize = GetFreeHeapSize() - config.initialSize;
		
		size_t decommitSize;
		// commit if > kDecommitThresholdPercentage is free
		if(freeSize * 100 > heapSize * kDecommitThresholdPercentage)
			decommitSize = int((freeSize * 100 - heapSize * kDecommitThresholdPercentage) / 100);
		else
			return;
		
		// don't trifle
		if(decommitSize < (size_t)kMinHeapIncrement)
			return;

		MMGC_LOCK(m_spinlock);

	restart:

		// search from the end of the free list so we decommit big blocks
		HeapBlock *freelist = freelists+kNumFreeLists-1;

		HeapBlock *endOfBigFreelists = &freelists[GetFreeListIndex(1)];

		for (; freelist >= endOfBigFreelists && decommitSize > 0; freelist--)
		{
			HeapBlock *block = freelist;
			while ((block = block->prev) != freelist && decommitSize > 0)
			{
				// decommitting already decommitted blocks doesn't help
				// temporary replacement for commented out conditional below
				GCAssert(block->size != 0);
				if(!block->committed /*|| block->size == 0*/)
					continue;

#ifdef MMGC_USE_VIRTUAL_MEMORY				
				RemoveFromList(block);
				if((size_t)block->size > decommitSize)
				{
					HeapBlock *newBlock = Split(block, (int)decommitSize);
					AddToFreeList(newBlock);
				}
				
				Region *region = AddrToRegion(block->baseAddr);
				if(config.trimVirtualMemory &&
				   freeSize * 100 > heapSize * kReleaseThresholdPercentage &&
					// if block is as big or bigger than a region, free the whole region
					block->baseAddr <= region->baseAddr && 
					region->reserveTop <= block->endAddr())
				{
					// trim block if they can span regions
					if(blocksSpanRegions)
					{
						if(block->baseAddr < region->baseAddr)
						{
							HeapBlock *newBlock = Split(block, int((region->baseAddr - block->baseAddr) / kBlockSize));
							AddToFreeList(block);
							block = newBlock;
						}
						if(block->endAddr() > region->reserveTop)
						{
							HeapBlock *newBlock = Split(block, int((region->reserveTop - block->baseAddr) / kBlockSize));
							AddToFreeList(newBlock);
						}
					}
					decommitSize -= block->size;
					RemoveBlock(block);
					goto restart;
				}
				else if(DecommitMemoryThatMaySpanRegions(block->baseAddr, block->size * kBlockSize))
				{
					block->committed = false;
					block->dirty = false;
					decommitSize -= block->size;
					if(config.verbose) {
						GCLog("decommitted %d page block from %p\n", block->size, block->baseAddr);
					}
				}
				else
				{
#ifdef _MAC
					// this can happen on mac where we release and re-reserve the memory and another thread may steal it from us
					RemoveBlock(block);
					goto restart;
#else
					// if the VM API's fail us bail
					VMPI_abort(); 
#endif
				}

				numDecommitted += block->size;

				// merge with previous/next if not in use and not committed
				HeapBlock *prev = block - block->sizePrevious;
				if(block->sizePrevious != 0 && !prev->committed && !prev->inUse()) {
					RemoveFromList(prev);

					prev->size += block->size;

					block->size = 0;
					block->sizePrevious = 0;
					block->baseAddr = 0;

					block = prev;
				}

				HeapBlock *next = block + block->size;
				if(next->size != 0 && !next->committed && !next->inUse()) {
					RemoveFromList(next);

					block->size += next->size;
				
					next->size = 0;
					next->sizePrevious = 0;
					next->baseAddr = 0;
				}
				
				next = block + block->size;
				next->sizePrevious = block->size;
				
				// add this block to the back of the bus to make sure we consume committed memory
				// first
				HeapBlock *backOfTheBus = &freelists[kNumFreeLists-1];
				HeapBlock *pointToInsert = backOfTheBus;
				while ((pointToInsert = pointToInsert->next) !=  backOfTheBus) {
					if (pointToInsert->size >= block->size && !pointToInsert->committed) {
						break;
					}
				}
				AddToFreeList(block, pointToInsert);

				// so we keep going through freelist properly
				block = freelist;
#else
				// if we aren't using mmap we can only do something if the block maps to a region
				// that is completely empty
				Region *region = AddrToRegion(block->baseAddr);
				if(block->baseAddr == region->baseAddr && // beginnings match
					region->commitTop == block->baseAddr + block->size*kBlockSize) {

					RemoveFromList(block);
					
					RemoveBlock(block);

					goto restart;
				}
#endif
			}
		}

		if(config.verbose)
			DumpHeapRep();
	}

	void GCHeap::RemoveBlock(HeapBlock *block)
	{	
		Region *region = AddrToRegion(block->baseAddr);
		
		GCAssert(region->baseAddr == block->baseAddr);
		GCAssert(region->reserveTop == block->endAddr());
		
		// update nextblock's sizePrevious
		HeapBlock *nextBlock = block + block->size;
		nextBlock->sizePrevious = block->sizePrevious;
		
		int newBlocksLen = blocksLen - block->size;

		bool need_sentinel = (block-blocks > 0);

		// Add space for the sentinel - the remaining blocks won't be contiguous
		if(need_sentinel)
			++newBlocksLen;

		// we're removing a region so re-allocate the blocks w/o the blocks for this region
		HeapBlock *newBlocks = new HeapBlock[newBlocksLen];
		
		// copy blocks before this block
		memcpy(newBlocks, blocks, (block - blocks) * sizeof(HeapBlock));
		
		size_t offset = block-blocks;

		if( need_sentinel ) {
			offset = block-blocks+1;

			HeapBlock* sentinel = newBlocks +(block-blocks+1);
			sentinel->baseAddr = NULL;
			sentinel->size = 0;<commit_msg>Fix build Turn off memstats fix for now, since it seems to be causing problems.<commit_after>/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: t; tab-width: 4 -*- */
/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is [Open Source Virtual Machine.].
 *
 * The Initial Developer of the Original Code is
 * Adobe System Incorporated.
 * Portions created by the Initial Developer are Copyright (C) 2004-2006
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *   Adobe AS3 Team
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

#include "MMgc.h"

#ifdef AVMPLUS_SAMPLER
namespace avmplus
{
	void recordAllocationSample(const void* item, size_t size);
	void recordDeallocationSample(const void* item, size_t size);	
}
#endif

namespace MMgc
{
	GCHeap *GCHeap::instance = NULL;

	/** The native VM page size (in bytes) for the current architecture */
	const int GCHeap::kNativePageSize = GCHeap::vmPageSize();

	void GCHeap::Init(GCHeapConfig& config)
	{
		GCAssert(instance == NULL);
		instance = new GCHeap(config);
	}

	void GCHeap::Destroy()
	{
		GCAssert(instance != NULL);
		delete instance;
		instance = NULL;
	}

	GCHeap::GCHeap(GCHeapConfig& c)
		: blocks(NULL),
		  blocksLen(0),
		  numDecommitted(0),
		  numAlloc(0),
		  config(c),
#ifdef MMGC_MEMORY_PROFILER
		  profiler(NULL),
		  signal(0),
#endif
	      spyFile(stdout),
		  hooksEnabled(false),
		  blocksSpanRegions(true), // TODO: see if making this false increases heap shrinkage opportunities
		  searchForOldestBlock(!blocksSpanRegions), 
		  mergeContiguousRegions(osSupportsRegionMerging())
	{		
		lastRegion  = 0;
		blocksLen   = 0;
		numAlloc    = 0;
		numDecommitted = 0;
		blocks      = NULL;

		// Initialize free lists
		HeapBlock *block = freelists;
		for (int i=0; i<kNumFreeLists; i++) {
			block->baseAddr     = NULL;
			block->size         = 0;
			block->sizePrevious = 0;
			block->prev         = block;
			block->next         = block;
			block->committed    = true;
			block->dirty 	    = true;
			block++;
		}
		

		// Create the initial heap
		ExpandHeap((int)config.initialSize);

		fixedMalloc._Init(this);

		instance = this;

		gcs = NULL;
		gcs_count = 0;
		status = kNormal;
		
		bool enableHooks = false;
		
#ifdef MMGC_MEMORY_PROFILER
		if(!config.enableProfiler) {
			const char *env = VMPI_getenv("MMGC_PROFILE");
			if(env && VMPI_strncmp(env, "1", 1) == 0)
				config.enableProfiler = true;
		}
		enableHooks = config.enableProfiler;
#endif
		
#ifdef MMGC_MEMORY_INFO
		// always track allocs in DEBUG builds
		enableHooks = true;
#endif
		
		if(enableHooks) {
#ifdef MMGC_MEMORY_PROFILER
			profiler = new MemoryProfiler();
			VMPI_WriteOnNamedSignal("MMgc::MemoryProfiler::DumpFatties", &signal);
#endif
			hooksEnabled = true; // set only after creating profiler	
		}
	}

	GCHeap::~GCHeap()
	{
		delete [] gcs;

		fixedMalloc._Destroy();

#ifdef _DEBUG
		if(numAlloc != 0)
		{
			for (unsigned int i=0; i<blocksLen; i++) 
			{
				HeapBlock *block = &blocks[i];
				if(block->inUse() && block->baseAddr)
				{
					GCDebugMsg(false, "Block 0x%x not freed\n", block->baseAddr);
#ifdef MMGC_MEMORY_PROFILER
					if(block->allocTrace)
						PrintStackTrace(block->allocTrace);
#endif
				}
			}	
			GCAssert(false);
		}
#endif
		
#ifdef MMGC_MEMORY_PROFILER
		hooksEnabled = false;
		if(profiler)
			delete profiler;
		profiler = NULL;
#endif

		FreeAll();
	}

	void* GCHeap::Alloc(int size, bool expand/*=true*/, bool zero/*true*/)
	{
		GCAssert(size > 0);

		char *baseAddr = 0;

		// nested block to keep memset/memory commit out of critical section
		{
			MMGC_LOCK(m_spinlock);

			HeapBlock *block = AllocBlock(size, zero);

			if (!block && expand) {
				ExpandHeapLocked(size);		
				block = AllocBlock(size, zero);
			}

			if (block) 
			{			
				GCAssert(block->size == size);
			
				numAlloc += size;

				// copy baseAddr to a stack variable to fix :
				// http://flashqa.macromedia.com/bugapp/detail.asp?ID=125938
				baseAddr = block->baseAddr;
			}
		}

		if(!baseAddr && expand) {
			Abort();
		}
		
		if(baseAddr)
		{
			// Zero out the memory, if requested to do so
			// FIXME: if we know that this memory was just committed we shouldn't do this
			// maybe zero should pass deeper into the system so that on initial expansions
			// and re-commit we avoid this, should be a noise optimization over all but
			// possibly a big win for startup
			if (zero) {
				VMPI_memset(baseAddr, 0, size * kBlockSize);
			}
		}
		
		return baseAddr;
	}

	void GCHeap::Free(void *item)
	{
		MMGC_LOCK(m_spinlock);

		HeapBlock *block = AddrToBlock(item);
		if (block) {
			// Update metrics
			GCAssert(numAlloc >= (unsigned int)block->size);
			numAlloc -= block->size;
			
			FreeBlock(block);
		}
	}

	
	size_t GCHeap::Size(const void *item)
	{
		MMGC_LOCK(m_spinlock);
		HeapBlock *block = AddrToBlock(item);
		return block->size;
	}

	void GCHeap::Decommit()
	{
		// keep at least initialSize free 
		if(!config.returnMemory || GetFreeHeapSize() <= config.initialSize)
			return;
		
		size_t heapSize = GetTotalHeapSize();
		size_t freeSize = GetFreeHeapSize() - config.initialSize;
		
		size_t decommitSize;
		// commit if > kDecommitThresholdPercentage is free
		if(freeSize * 100 > heapSize * kDecommitThresholdPercentage)
			decommitSize = int((freeSize * 100 - heapSize * kDecommitThresholdPercentage) / 100);
		else
			return;
		
		// don't trifle
		if(decommitSize < (size_t)kMinHeapIncrement)
			return;

		MMGC_LOCK(m_spinlock);

	restart:

		// search from the end of the free list so we decommit big blocks
		HeapBlock *freelist = freelists+kNumFreeLists-1;

		HeapBlock *endOfBigFreelists = &freelists[GetFreeListIndex(1)];

		for (; freelist >= endOfBigFreelists && decommitSize > 0; freelist--)
		{
			HeapBlock *block = freelist;
			while ((block = block->prev) != freelist && decommitSize > 0)
			{
				// decommitting already decommitted blocks doesn't help
				// temporary replacement for commented out conditional below
				GCAssert(block->size != 0);
				if(!block->committed /*|| block->size == 0*/)
					continue;

#ifdef MMGC_USE_VIRTUAL_MEMORY				
				RemoveFromList(block);
				if((size_t)block->size > decommitSize)
				{
					HeapBlock *newBlock = Split(block, (int)decommitSize);
					AddToFreeList(newBlock);
				}
				
				Region *region = AddrToRegion(block->baseAddr);
				if(config.trimVirtualMemory &&
				   freeSize * 100 > heapSize * kReleaseThresholdPercentage &&
					// if block is as big or bigger than a region, free the whole region
					block->baseAddr <= region->baseAddr && 
					region->reserveTop <= block->endAddr())
				{
					// trim block if they can span regions
					if(blocksSpanRegions)
					{
						if(block->baseAddr < region->baseAddr)
						{
							HeapBlock *newBlock = Split(block, int((region->baseAddr - block->baseAddr) / kBlockSize));
							AddToFreeList(block);
							block = newBlock;
						}
						if(block->endAddr() > region->reserveTop)
						{
							HeapBlock *newBlock = Split(block, int((region->reserveTop - block->baseAddr) / kBlockSize));
							AddToFreeList(newBlock);
						}
					}
					decommitSize -= block->size;
					RemoveBlock(block);
					goto restart;
				}
				else if(DecommitMemoryThatMaySpanRegions(block->baseAddr, block->size * kBlockSize))
				{
					block->committed = false;
					block->dirty = false;
					decommitSize -= block->size;
					if(config.verbose) {
						GCLog("decommitted %d page block from %p\n", block->size, block->baseAddr);
					}
				}
				else
				{
#ifdef _MAC
					// this can happen on mac where we release and re-reserve the memory and another thread may steal it from us
					RemoveBlock(block);
					goto restart;
#else
					// if the VM API's fail us bail
					VMPI_abort(); 
#endif
				}

				numDecommitted += block->size;

				// merge with previous/next if not in use and not committed
				HeapBlock *prev = block - block->sizePrevious;
				if(block->sizePrevious != 0 && !prev->committed && !prev->inUse()) {
					RemoveFromList(prev);

					prev->size += block->size;

					block->size = 0;
					block->sizePrevious = 0;
					block->baseAddr = 0;

					block = prev;
				}

				HeapBlock *next = block + block->size;
				if(next->size != 0 && !next->committed && !next->inUse()) {
					RemoveFromList(next);

					block->size += next->size;
				
					next->size = 0;
					next->sizePrevious = 0;
					next->baseAddr = 0;
				}
				
				next = block + block->size;
				next->sizePrevious = block->size;
				
				// add this block to the back of the bus to make sure we consume committed memory
				// first
				HeapBlock *backOfTheBus = &freelists[kNumFreeLists-1];
				HeapBlock *pointToInsert = backOfTheBus;
				while ((pointToInsert = pointToInsert->next) !=  backOfTheBus) {
					if (pointToInsert->size >= block->size && !pointToInsert->committed) {
						break;
					}
				}
				AddToFreeList(block, pointToInsert);

				// so we keep going through freelist properly
				block = freelist;
#else
				// if we aren't using mmap we can only do something if the block maps to a region
				// that is completely empty
				Region *region = AddrToRegion(block->baseAddr);
				if(block->baseAddr == region->baseAddr && // beginnings match
					region->commitTop == block->baseAddr + block->size*kBlockSize) {

					RemoveFromList(block);
					
					RemoveBlock(block);

					goto restart;
				}
#endif
			}
		}

		if(config.verbose)
			DumpHeapRep();
	}

	void GCHeap::RemoveBlock(HeapBlock *block)
	{	
		Region *region = AddrToRegion(block->baseAddr);
		
		GCAssert(region->baseAddr == block->baseAddr);
		GCAssert(region->reserveTop == block->endAddr());
		
		// update nextblock's sizePrevious
		HeapBlock *nextBlock = block + block->size;
		nextBlock->sizePrevious = block->sizePrevious;
		
		int newBlocksLen = blocksLen - block->size;

		//bool need_sentinel = (block-blocks > 0);
		bool need_sentinel = false;

		// Add space for the sentinel - the remaining blocks won't be contiguous
		if(need_sentinel)
			++newBlocksLen;

		// we're removing a region so re-allocate the blocks w/o the blocks for this region
		HeapBlock *newBlocks = new HeapBlock[newBlocksLen];
		
		// copy blocks before this block
		memcpy(newBlocks, blocks, (block - blocks) * sizeof(HeapBlock));
		
		size_t offset = block-blocks;

		if( need_sentinel ) {
			offset = block-blocks+1;

			HeapBlock* sentinel = newBlocks +(block-blocks+1);
			sentinel->baseAddr = NULL;
			sentinel->size = 0;
{'fraction_non_alphanumeric': 0.11473316878032173, 'fraction_numerical': 0.006809090135330666, 'mean_word_length': 3.1168535388927823, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 199, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7609939', 'n_tokens_mistral': 9204, 'n_tokens_neox': 7536, 'n_words': 2194}