<commit_before>#include "mustela.hpp"
#include <iostream>

using namespace mustela;

constexpr size_t RECORD_SIZE = NODE_PID_SIZE + 1;
// Use store fixed-size records of NODE_PID_SIZE page + 1-byte count

// TODO - use variable-encoding for count, leave free worst-case free bytes at the end of page

static size_t get_records_count(Pid count){
	return ((count + 254) / 255);
}

void MergablePageCache::clear(){
	cache.clear();
	record_count = 0;
	size_index.clear();
}

void MergablePageCache::add_to_size_index(Pid page, Pid count){
	auto & sitem = size_index[count];
	bool ins = sitem.insert(page).second;
	ass(ins, "Page is twice in size_index");
}
void MergablePageCache::remove_from_size_index(Pid page, Pid count){
	auto siit = size_index.find(count);
	ass(siit != size_index.end(), "Size index find count failed");
	ass(siit->second.erase(page) == 1, "Size index erase page failed");
	if(siit->second.empty())
		size_index.erase(siit);
}

void MergablePageCache::add_to_cache(Pid page, Pid count){
//	if( count == 255 )
//		std::cerr << "MergablePageCache::add_to_cache i=" << int(update_index) << " " << page << ":" << count << std::endl;
	auto it = cache.lower_bound(page);
	ass(it == cache.end() || it->first != page, "adding existing page to cache");
	if(it != cache.end() && it->first == page + count){
		if( update_index)
			remove_from_size_index(it->first, it->second);
		count += it->second;
		record_count -= get_records_count(it->second);
		it = cache.erase(it);
	}
	if( it != cache.begin() ){
		--it;
		if( it->first + it->second == page){
			if( update_index)
				remove_from_size_index(it->first, it->second);
			page = it->first;
			count += it->second;
			record_count -= get_records_count(it->second);
			it = cache.erase(it);
		}
	}
	record_count += get_records_count(count);
	cache.insert(std::make_pair(page, count));
	if( update_index)
		add_to_size_index(page, count);
}

void MergablePageCache::remove_from_cache(Pid page, Pid count){
	auto it = cache.find(page);
	ass(it != cache.end() && it->second >= count, "invalid remove from cache");
	if( update_index )
		remove_from_size_index(it->first, it->second);
	if( count == it->second ){
		record_count -= get_records_count(it->second);
		cache.erase(it);
		return;
	}
	auto old_count = it->second;
	record_count -= get_records_count(old_count);
	cache.erase(it);
	old_count -= count;
	page += count;
	if( update_index )
		add_to_size_index(page, old_count);
	record_count += get_records_count(old_count);
	cache.insert(std::make_pair(page, old_count));
}
Pid MergablePageCache::get_free_page(Pid contigous_count){
	auto siit = size_index.lower_bound(contigous_count);
	if( siit == size_index.end() )
		return 0;
	Pid pa = *(siit->second.begin());
	remove_from_cache(pa, contigous_count);
	ass(pa >= META_PAGES_COUNT, "Meta somehow got into freelist"); // TODO - constant
	// TODO - check tid of the page?
	return pa;
}
Pid MergablePageCache::defrag_end(Pid page_count){
	if( cache.empty() )
		return 0;
	auto fit = cache.end();<commit_msg>tiny tweak to free list for better chance of defrag<commit_after>#include "mustela.hpp"
#include <iostream>

using namespace mustela;

constexpr size_t RECORD_SIZE = NODE_PID_SIZE + 1;
// Use store fixed-size records of NODE_PID_SIZE page + 1-byte count

// TODO - use variable-encoding for count, leave free worst-case free bytes at the end of page

static size_t get_records_count(Pid count){
	return ((count + 254) / 255);
}

void MergablePageCache::clear(){
	cache.clear();
	record_count = 0;
	size_index.clear();
}

void MergablePageCache::add_to_size_index(Pid page, Pid count){
	auto & sitem = size_index[count];
	bool ins = sitem.insert(page).second;
	ass(ins, "Page is twice in size_index");
}
void MergablePageCache::remove_from_size_index(Pid page, Pid count){
	auto siit = size_index.find(count);
	ass(siit != size_index.end(), "Size index find count failed");
	ass(siit->second.erase(page) == 1, "Size index erase page failed");
	if(siit->second.empty())
		size_index.erase(siit);
}

void MergablePageCache::add_to_cache(Pid page, Pid count){
//	if( count == 255 )
//		std::cerr << "MergablePageCache::add_to_cache i=" << int(update_index) << " " << page << ":" << count << std::endl;
	auto it = cache.lower_bound(page);
	ass(it == cache.end() || it->first != page, "adding existing page to cache");
	if(it != cache.end() && it->first == page + count){
		if( update_index)
			remove_from_size_index(it->first, it->second);
		count += it->second;
		record_count -= get_records_count(it->second);
		it = cache.erase(it);
	}
	if( it != cache.begin() ){
		--it;
		if( it->first + it->second == page){
			if( update_index)
				remove_from_size_index(it->first, it->second);
			page = it->first;
			count += it->second;
			record_count -= get_records_count(it->second);
			it = cache.erase(it);
		}
	}
	record_count += get_records_count(count);
	cache.insert(std::make_pair(page, count));
	if( update_index)
		add_to_size_index(page, count);
}

void MergablePageCache::remove_from_cache(Pid page, Pid count){
	auto it = cache.find(page);
	ass(it != cache.end() && it->second >= count, "invalid remove from cache");
	if( update_index )
		remove_from_size_index(it->first, it->second);
	if( count == it->second ){
		record_count -= get_records_count(it->second);
		cache.erase(it);
		return;
	}
	auto old_count = it->second;
	record_count -= get_records_count(old_count);
	cache.erase(it);
	old_count -= count;
	page += count;
	if( update_index )
		add_to_size_index(page, old_count);
	record_count += get_records_count(old_count);
	cache.insert(std::make_pair(page, old_count));
}
Pid MergablePageCache::get_free_page(Pid contigous_count){
	auto siit = size_index.lower_bound(contigous_count);
	if( siit == size_index.end() )
		return 0;
	Pid pa = *(siit->second.begin());
	if(contigous_count == 1)
		pa = cache.begin()->first;
	remove_from_cache(pa, contigous_count);
	ass(pa >= META_PAGES_COUNT, "Meta somehow got into freelist"); // TODO - constant
	// TODO - check tid of the page?
	return pa;
}
Pid MergablePageCache::defrag_end(Pid page_count){
	if( cache.empty() )
		return 0;
	auto fit = cache.end();
{'fraction_non_alphanumeric': 0.15533822330888344, 'fraction_numerical': 0.005052974735126324, 'mean_word_length': 4.849380362249762, 'pattern_counts': {'":': 2, '<': 33, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5298432', 'n_tokens_mistral': 2342, 'n_tokens_neox': 2240, 'n_words': 510}