<commit_before>#include "dds_stream.hpp"

#include "gl_util.hpp"

#include "thirdparty/shaun/sweeper.hpp"
#include "thirdparty/shaun/parser.hpp"

#include <iostream>
#include <fstream>
#include <algorithm>
#include <cmath>

using namespace std;

GLenum DDSFormatToGL(DDSLoader::Format format)
{
	switch (format)
	{
		case DDSLoader::Format::BC1:       return GL_COMPRESSED_RGBA_S3TC_DXT1_EXT;
		case DDSLoader::Format::BC1_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT;
		case DDSLoader::Format::BC2:       return GL_COMPRESSED_RGBA_S3TC_DXT3_EXT;
		case DDSLoader::Format::BC2_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT;
		case DDSLoader::Format::BC3:       return GL_COMPRESSED_RGBA_S3TC_DXT5_EXT;
		case DDSLoader::Format::BC3_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT;
		case DDSLoader::Format::BC4:       return GL_COMPRESSED_RED_RGTC1;
		case DDSLoader::Format::BC4_SIGNED:return GL_COMPRESSED_SIGNED_RED_RGTC1;
		case DDSLoader::Format::BC5:       return GL_COMPRESSED_RG_RGTC2;
		case DDSLoader::Format::BC5_SIGNED:return GL_COMPRESSED_SIGNED_RG_RGTC2;
		case DDSLoader::Format::BC6:       return GL_COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT;
		case DDSLoader::Format::BC6_SIGNED:return GL_COMPRESSED_RGB_BPTC_SIGNED_FLOAT;
		case DDSLoader::Format::BC7:       return GL_COMPRESSED_RGBA_BPTC_UNORM;
		case DDSLoader::Format::BC7_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_BPTC_UNORM;

	}
	return 0;
}

void DDSStreamer::init(int pageSize, int numPages, int maxSize)
{
	_maxSize = maxSize;
	_pageSize = pageSize;
	_numPages = numPages;
	int pboSize = pageSize*numPages;
	glCreateBuffers(1, &_pbo);
	GLbitfield storageFlags = GL_MAP_WRITE_BIT|GL_MAP_PERSISTENT_BIT;
#ifdef USE_COHERENT_MAPPING
	storageFlags = storageFlags | GL_MAP_COHERENT_BIT;
#endif
	glNamedBufferStorage(_pbo, pboSize, nullptr, storageFlags);
	GLbitfield mapFlags = storageFlags;
#ifndef USE_COHERENT_MAPPING
	mapFlags = mapFlags | GL_MAP_FLUSH_EXPLICIT_BIT;
#endif
	_pboPtr = glMapNamedBufferRange(
		_pbo, 0, pboSize, mapFlags);

	_usedPages.resize(numPages, false);
	_pageFences.resize(numPages);

	_t = thread([this]{
		while (true)
		{
			LoadInfo info{};
			{
				unique_lock<mutex> lk(_mtx);
				_cond.wait(lk, [this]{ return _killThread || !_loadInfoQueue.empty();});
				if (_killThread) return;
				info = _loadInfoQueue[0];
				_loadInfoQueue.erase(_loadInfoQueue.begin());
			}

			LoadData data = load(info);

			{
				lock_guard<mutex> lk(_dataMtx);
				_loadData.push_back(data);
			}
		}
	});
}

DDSStreamer::~DDSStreamer()
{
	if (_pbo)
	{
		glUnmapNamedBuffer(_pbo);
		glDeleteBuffers(1, &_pbo);
	}

	if (_t.joinable())
	{
		{
			lock_guard<mutex> lk(_mtx);
			_killThread = true;
		}
		_cond.notify_one();
		_t.join();
	}
}

struct TexInfo
{
	int size = 0;
	int levels = 0;
	string prefix = "";
	string separator = "";
	string suffix = "";
	bool rowColumnOrder = false;
};

TexInfo parseInfoFile(const string &filename, int maxSize)
{
	ifstream in(filename.c_str(), ios::in | ios::binary);
	if (!in) return {};

	string contents;
	in.seekg(0, ios::end);
	contents.resize(in.tellg());
	in.seekg(0, ios::beg);
	in.read(&contents[0], contents.size());

	try
	{
		shaun::parser p{};
		shaun::object obj = p.parse(contents.c_str());
		shaun::sweeper swp(&obj);

		TexInfo info{};
		info.size = swp("size").value<shaun::number>();
		info.levels = swp("levels").value<shaun::number>();
		info.prefix = (string)swp("prefix").value<shaun::string>();
		info.separator = (string)swp("separator").value<shaun::string>();
		info.suffix = (string)swp("suffix").value<shaun::string>();
		info.rowColumnOrder = swp("row_column_order").value<shaun::boolean>();

		if (maxSize)
		{
			int maxRows = maxSize/(info.size*2);
			int maxLevel = (int)floor(log2(maxRows))+1;
			info.levels = max(1,min(info.levels, maxLevel));
		}
		return info;
	} 
	catch (shaun::parse_error &e)
	{
		cout << e << endl;
		return {};
	}
}

DDSStreamer::Handle DDSStreamer::createTex(const string &filename)
{
	// Get info file
	TexInfo info = parseInfoFile(filename + "/info.sn", _maxSize);

	// Check if file exists or is valid
	if (info.levels == 0) return 0;

	const string tailFile = "/level0/0_0.DDS";

	// Gen texture
	const Handle h = genHandle();
	GLuint id;
	glCreateTextures(GL_TEXTURE_2D, 1, &id);
	_texs.insert(make_pair(h, StreamTexture(id)));

	// Tail loader
	DDSLoader tailLoader = DDSLoader(filename+tailFile);

	// Storage
	const int width = info.size<<(info.levels-1);
	const int height = width/2;
	const GLenum format = DDSFormatToGL(tailLoader.getFormat());
	glTextureStorage2D(id, mipmapCount(width), format, width, height);

	// Gen jobs
	vector<LoadInfo> jobs;

	// Tail mipmaps (level0)
	const int tailMips = mipmapCount(info.size);
	for (int i=tailMips-1;i>=0;--i)
	{
		LoadInfo tailInfo{};
		tailInfo.handle = h;
		tailInfo.loader = tailLoader;
		tailInfo.fileLevel = i;<commit_msg>Fix texture tail file name<commit_after>#include "dds_stream.hpp"

#include "gl_util.hpp"

#include "thirdparty/shaun/sweeper.hpp"
#include "thirdparty/shaun/parser.hpp"

#include <iostream>
#include <fstream>
#include <algorithm>
#include <cmath>

using namespace std;

GLenum DDSFormatToGL(DDSLoader::Format format)
{
	switch (format)
	{
		case DDSLoader::Format::BC1:       return GL_COMPRESSED_RGBA_S3TC_DXT1_EXT;
		case DDSLoader::Format::BC1_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT;
		case DDSLoader::Format::BC2:       return GL_COMPRESSED_RGBA_S3TC_DXT3_EXT;
		case DDSLoader::Format::BC2_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT;
		case DDSLoader::Format::BC3:       return GL_COMPRESSED_RGBA_S3TC_DXT5_EXT;
		case DDSLoader::Format::BC3_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT;
		case DDSLoader::Format::BC4:       return GL_COMPRESSED_RED_RGTC1;
		case DDSLoader::Format::BC4_SIGNED:return GL_COMPRESSED_SIGNED_RED_RGTC1;
		case DDSLoader::Format::BC5:       return GL_COMPRESSED_RG_RGTC2;
		case DDSLoader::Format::BC5_SIGNED:return GL_COMPRESSED_SIGNED_RG_RGTC2;
		case DDSLoader::Format::BC6:       return GL_COMPRESSED_RGB_BPTC_UNSIGNED_FLOAT;
		case DDSLoader::Format::BC6_SIGNED:return GL_COMPRESSED_RGB_BPTC_SIGNED_FLOAT;
		case DDSLoader::Format::BC7:       return GL_COMPRESSED_RGBA_BPTC_UNORM;
		case DDSLoader::Format::BC7_SRGB:  return GL_COMPRESSED_SRGB_ALPHA_BPTC_UNORM;

	}
	return 0;
}

void DDSStreamer::init(int pageSize, int numPages, int maxSize)
{
	_maxSize = maxSize;
	_pageSize = pageSize;
	_numPages = numPages;
	int pboSize = pageSize*numPages;
	glCreateBuffers(1, &_pbo);
	GLbitfield storageFlags = GL_MAP_WRITE_BIT|GL_MAP_PERSISTENT_BIT;
#ifdef USE_COHERENT_MAPPING
	storageFlags = storageFlags | GL_MAP_COHERENT_BIT;
#endif
	glNamedBufferStorage(_pbo, pboSize, nullptr, storageFlags);
	GLbitfield mapFlags = storageFlags;
#ifndef USE_COHERENT_MAPPING
	mapFlags = mapFlags | GL_MAP_FLUSH_EXPLICIT_BIT;
#endif
	_pboPtr = glMapNamedBufferRange(
		_pbo, 0, pboSize, mapFlags);

	_usedPages.resize(numPages, false);
	_pageFences.resize(numPages);

	_t = thread([this]{
		while (true)
		{
			LoadInfo info{};
			{
				unique_lock<mutex> lk(_mtx);
				_cond.wait(lk, [this]{ return _killThread || !_loadInfoQueue.empty();});
				if (_killThread) return;
				info = _loadInfoQueue[0];
				_loadInfoQueue.erase(_loadInfoQueue.begin());
			}

			LoadData data = load(info);

			{
				lock_guard<mutex> lk(_dataMtx);
				_loadData.push_back(data);
			}
		}
	});
}

DDSStreamer::~DDSStreamer()
{
	if (_pbo)
	{
		glUnmapNamedBuffer(_pbo);
		glDeleteBuffers(1, &_pbo);
	}

	if (_t.joinable())
	{
		{
			lock_guard<mutex> lk(_mtx);
			_killThread = true;
		}
		_cond.notify_one();
		_t.join();
	}
}

struct TexInfo
{
	int size = 0;
	int levels = 0;
	string prefix = "";
	string separator = "";
	string suffix = "";
	bool rowColumnOrder = false;
};

TexInfo parseInfoFile(const string &filename, int maxSize)
{
	ifstream in(filename.c_str(), ios::in | ios::binary);
	if (!in) return {};

	string contents;
	in.seekg(0, ios::end);
	contents.resize(in.tellg());
	in.seekg(0, ios::beg);
	in.read(&contents[0], contents.size());

	try
	{
		shaun::parser p{};
		shaun::object obj = p.parse(contents.c_str());
		shaun::sweeper swp(&obj);

		TexInfo info{};
		info.size = swp("size").value<shaun::number>();
		info.levels = swp("levels").value<shaun::number>();
		info.prefix = (string)swp("prefix").value<shaun::string>();
		info.separator = (string)swp("separator").value<shaun::string>();
		info.suffix = (string)swp("suffix").value<shaun::string>();
		info.rowColumnOrder = swp("row_column_order").value<shaun::boolean>();

		if (maxSize)
		{
			int maxRows = maxSize/(info.size*2);
			int maxLevel = (int)floor(log2(maxRows))+1;
			info.levels = max(1,min(info.levels, maxLevel));
		}
		return info;
	} 
	catch (shaun::parse_error &e)
	{
		cout << e << endl;
		return {};
	}
}

DDSStreamer::Handle DDSStreamer::createTex(const string &filename)
{
	// Get info file
	TexInfo info = parseInfoFile(filename + "/info.sn", _maxSize);

	// Check if file exists or is valid
	if (info.levels == 0) return 0;

	const string tailFile = "/level0/" + info.prefix + "0" + info.separator + "0" + info.suffix;

	// Gen texture
	const Handle h = genHandle();
	GLuint id;
	glCreateTextures(GL_TEXTURE_2D, 1, &id);
	_texs.insert(make_pair(h, StreamTexture(id)));

	// Tail loader
	DDSLoader tailLoader = DDSLoader(filename+tailFile);

	// Storage
	const int width = info.size<<(info.levels-1);
	const int height = width/2;
	const GLenum format = DDSFormatToGL(tailLoader.getFormat());
	glTextureStorage2D(id, mipmapCount(width), format, width, height);

	// Gen jobs
	vector<LoadInfo> jobs;

	// Tail mipmaps (level0)
	const int tailMips = mipmapCount(info.size);
	for (int i=tailMips-1;i>=0;--i)
	{
		LoadInfo tailInfo{};
		tailInfo.handle = h;
		tailInfo.loader = tailLoader;
		tailInfo.fileLevel = i;
{'fraction_non_alphanumeric': 0.13770992366412213, 'fraction_numerical': 0.011603053435114504, 'mean_word_length': 5.137414116177389, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3782600', 'n_tokens_mistral': 4103, 'n_tokens_neox': 3825, 'n_words': 676}