<commit_before>/* Copyright (c) 2017-2018 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include "device.hpp"
#include "format.hpp"
#include "thread_group.hpp"
#include "type_to_string.hpp"
#include "quirks.hpp"
#include "enum_cast.hpp"
#include <algorithm>
#include <string.h>

#ifdef VULKAN_MT
#define LOCK() std::lock_guard<std::mutex> holder__{lock.lock}
#define DRAIN_FRAME_LOCK() \
	std::unique_lock<std::mutex> holder__{lock.lock}; \
	lock.cond.wait(holder__, [&]() { \
		return lock.counter == 0; \
	})
#else
#define LOCK() ((void)0)
#define DRAIN_FRAME_LOCK() VK_ASSERT(lock.counter == 0)
#endif

using namespace std;
using namespace Util;

namespace Vulkan
{
Device::Device()
    : framebuffer_allocator(this)
    , transient_allocator(this)
	, physical_allocator(this)
	, shader_manager(this)
	, texture_manager(this)
{
	cookie.store(0);

	ThreadGroup::get_global();
	ThreadGroup::register_main_thread();
}

Semaphore Device::request_semaphore()
{
	LOCK();
	auto semaphore = managers.semaphore.request_cleared_semaphore();
	auto ptr = make_handle<SemaphoreHolder>(this, semaphore, false);
	return ptr;
}

#ifndef _WIN32
Semaphore Device::request_imported_semaphore(int fd, VkExternalSemaphoreHandleTypeFlagBitsKHR handle_type)
{
	LOCK();
	if (!ext.supports_external)
		return {};

	VkExternalSemaphorePropertiesKHR props = { VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES_KHR };
	VkPhysicalDeviceExternalSemaphoreInfoKHR info = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO_KHR };
	info.handleType = handle_type;

	vkGetPhysicalDeviceExternalSemaphorePropertiesKHR(gpu, &info, &props);
	if ((props.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT_KHR) == 0)
		return Semaphore(nullptr);

	auto semaphore = managers.semaphore.request_cleared_semaphore();

	VkImportSemaphoreFdInfoKHR import = { VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR };
	import.fd = fd;
	import.semaphore = semaphore;
	import.handleType = handle_type;
	import.flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT_KHR;
	auto ptr = make_handle<SemaphoreHolder>(this, semaphore, false);

	if (vkImportSemaphoreFdKHR(device, &import) != VK_SUCCESS)
		return Semaphore(nullptr);

	ptr->signal_external();
	ptr->destroy_on_consume();
	return ptr;
}
#endif

void Device::add_wait_semaphore(CommandBuffer::Type type, Semaphore semaphore, VkPipelineStageFlags stages, bool flush)
{
	LOCK();
	add_wait_semaphore_nolock(type, semaphore, stages, flush);
}

void Device::add_wait_semaphore_nolock(CommandBuffer::Type type, Semaphore semaphore, VkPipelineStageFlags stages,
                                       bool flush)
{
	VK_ASSERT(stages != 0);
	if (flush)
		flush_frame(type);
	auto &data = get_queue_data(type);

#ifdef VULKAN_DEBUG
	for (auto &sem : data.wait_semaphores)
		VK_ASSERT(sem.get() != semaphore.get());
#endif

	semaphore->signal_pending_wait();
	data.wait_semaphores.push_back(semaphore);
	data.wait_stages.push_back(stages);
	data.need_fence = true;

	// Sanity check.
	VK_ASSERT(data.wait_semaphores.size() < 1024);
}

void *Device::map_host_buffer(Buffer &buffer, MemoryAccessFlags access)
{
	void *host = managers.memory.map_memory(&buffer.get_allocation(), access);
	return host;
}

void Device::unmap_host_buffer(const Buffer &buffer)
{
	managers.memory.unmap_memory(buffer.get_allocation());
}

Shader *Device::request_shader(const uint32_t *data, size_t size)
{
	Util::Hasher hasher;
	hasher.data(data, size);

	auto hash = hasher.get();
	auto *ret = shaders.find(hash);
	if (!ret)
		ret = shaders.insert(hash, make_unique<Shader>(hash, this, data, size));
	return ret;
}

bool Device::enqueue_create_shader_module(Fossilize::Hash hash, unsigned, const VkShaderModuleCreateInfo *create_info, VkShaderModule *module)
{
	auto *ret = shaders.insert(hash, make_unique<Shader>(hash, this, create_info->pCode, create_info->codeSize));
	*module = ret->get_module();
	replayer_state.shader_map[*module] = ret;
	return true;
}

bool Device::enqueue_create_compute_pipeline(Fossilize::Hash hash, unsigned,
                                             const VkComputePipelineCreateInfo *create_info, VkPipeline *pipeline)
{
	// Find the Shader* associated with this VkShaderModule and just use that.
	auto itr = replayer_state.shader_map.find(create_info->stage.module);
	if (itr == end(replayer_state.shader_map))
		return false;

	auto *ret = request_program(itr->second);
	*pipeline = ret->get_compute_pipeline();
	return true;
}

Program *Device::request_program(Vulkan::Shader *compute)
{
	Util::Hasher hasher;
	hasher.u64(compute->get_hash());

	auto hash = hasher.get();
	auto *ret = programs.find(hash);
	if (!ret)
		ret = programs.insert(hash, make_unique<Program>(hash, this, compute));
	return ret;
}

Program *Device::request_program(const uint32_t *compute_data, size_t compute_size)
{
	auto *compute = request_shader(compute_data, compute_size);
	return request_program(compute);
}

bool Device::enqueue_create_graphics_pipeline(Fossilize::Hash hash, unsigned,
                                              const VkGraphicsPipelineCreateInfo *create_info, VkPipeline *pipeline)
{
	auto info = *create_info;

	if (info.stageCount != 2)
		return false;
	if (info.pStages[0].stage != VK_SHADER_STAGE_VERTEX_BIT)
		return false;
	if (info.pStages[1].stage != VK_SHADER_STAGE_FRAGMENT_BIT)
		return false;

	// Find the Shader* associated with this VkShaderModule and just use that.
	auto vertex_itr = replayer_state.shader_map.find(info.pStages[0].module);
	if (vertex_itr == end(replayer_state.shader_map))
		return false;

	// Find the Shader* associated with this VkShaderModule and just use that.
	auto fragment_itr = replayer_state.shader_map.find(info.pStages[1].module);
	if (fragment_itr == end(replayer_state.shader_map))
		return false;

	auto *ret = request_program(vertex_itr->second, fragment_itr->second);

	// The layout is dummy, resolve it here.
	info.layout = ret->get_pipeline_layout()->get_layout();

	get_state_recorder().register_graphics_pipeline(hash, info);
	LOGI("Creating graphics pipeline.\n");
	VkResult res = vkCreateGraphicsPipelines(device, pipeline_cache, 1, &info, nullptr, pipeline);
	if (res != VK_SUCCESS)
		LOGE("Failed to create graphics pipeline!\n");
	*pipeline = ret->add_graphics_pipeline(hash, *pipeline);
	return true;
}

Program *Device::request_program(Vulkan::Shader *vertex, Vulkan::Shader *fragment)
{
	Util::Hasher hasher;
	hasher.u64(vertex->get_hash());
	hasher.u64(fragment->get_hash());

	auto hash = hasher.get();
	auto *ret = programs.find(hash);

	if (!ret)
		ret = programs.insert(hash, make_unique<Program>(hash, this, vertex, fragment));
	return ret;
}

Program *Device::request_program(const uint32_t *vertex_data, size_t vertex_size, const uint32_t *fragment_data,
                                 size_t fragment_size)
{
	auto *vertex = request_shader(vertex_data, vertex_size);
	auto *fragment = request_shader(fragment_data, fragment_size);
	return request_program(vertex, fragment);
}

PipelineLayout *Device::request_pipeline_layout(const CombinedResourceLayout &layout)
{
	Hasher h;
	h.data(reinterpret_cast<const uint32_t *>(layout.sets), sizeof(layout.sets));
	h.data(&layout.stages_for_bindings[0][0], sizeof(layout.stages_for_bindings));
	h.data(reinterpret_cast<const uint32_t *>(layout.ranges), sizeof(layout.ranges));
	h.u32(layout.attribute_mask);
	h.u32(layout.render_target_mask);

	auto hash = h.get();
	auto *ret = pipeline_layouts.find(hash);
	if (!ret)
		ret = pipeline_layouts.insert(hash, make_unique<PipelineLayout>(hash, this, layout));
	return ret;
}

DescriptorSetAllocator *Device::request_descriptor_set_allocator(const DescriptorSetLayout &layout, const uint32_t *stages_for_bindings)
{
	Hasher h;
	h.data(reinterpret_cast<const uint32_t *>(&layout), sizeof(layout));
	h.data(stages_for_bindings, sizeof(uint32_t) * VULKAN_NUM_BINDINGS);
	auto hash = h.get();

	auto *ret = descriptor_set_allocators.find(hash);
	if (!ret)
		ret = descriptor_set_allocators.insert(hash, make_unique<DescriptorSetAllocator>(hash, this, layout, stages_for_bindings));
	return ret;
}

void Device::bake_program(Program &program)
{
	CombinedResourceLayout layout;
	if (program.get_shader(ShaderStage::Vertex))
		layout.attribute_mask = program.get_shader(ShaderStage::Vertex)->get_layout().input_mask;
	if (program.get_shader(ShaderStage::Fragment))
		layout.render_target_mask = program.get_shader(ShaderStage::Fragment)->get_layout().output_mask;

	layout.descriptor_set_mask = 0;

	for (unsigned i = 0; i < static_cast<unsigned>(ShaderStage::Count); i++)
	{
		auto *shader = program.get_shader(static_cast<ShaderStage>(i));
		if (!shader)
			continue;

		uint32_t stage_mask = 1u << i;

		auto &shader_layout = shader->get_layout();
		for (unsigned set = 0; set < VULKAN_NUM_DESCRIPTOR_SETS; set++)
		{
			layout.sets[set].sampled_image_mask |= shader_layout.sets[set].sampled_image_mask;
			layout.sets[set].storage_image_mask |= shader_layout.sets[set].storage_image_mask;
			layout.sets[set].uniform_buffer_mask |= shader_layout.sets[set].uniform_buffer_mask;
			layout.sets[set].storage_buffer_mask |= shader_layout.sets[set].storage_buffer_mask;
			layout.sets[set].sampled_buffer_mask |= shader_layout.sets[set].sampled_buffer_mask;
			layout.sets[set].input_attachment_mask |= shader_layout.sets[set].input_attachment_mask;
			layout.sets[set].sampler_mask |= shader_layout.sets[set].sampler_mask;
			layout.sets[set].separate_image_mask |= shader_layout.sets[set].separate_image_mask;
			layout.sets[set].fp_mask |= shader_layout.sets[set].fp_mask;

			uint32_t active_binds =
					shader_layout.sets[set].sampled_image_mask |
					shader_layout.sets[set].storage_image_mask |
					shader_layout.sets[set].uniform_buffer_mask|
					shader_layout.sets[set].storage_buffer_mask |
					shader_layout.sets[set].sampled_buffer_mask |
					shader_layout.sets[set].input_attachment_mask |
					shader_layout.sets[set].sampler_mask |
					shader_layout.sets[set].separate_image_mask;

			if (active_binds)
				layout.stages_for_sets[set] |= stage_mask;

			for_each_bit(active_binds, [&](uint32_t bit) {
				layout.stages_for_bindings[set][bit] |= stage_mask;
			});
		}

		layout.ranges[i].stageFlags = 1u << i;
		layout.ranges[i].offset = shader_layout.push_constant_offset;
		layout.ranges[i].size = shader_layout.push_constant_range;
	}

	for (unsigned i = 0; i < VULKAN_NUM_DESCRIPTOR_SETS; i++)
	{
		if (layout.stages_for_sets[i] != 0)
			layout.descriptor_set_mask |= 1u << i;
	}

	Hasher h;
	h.data(reinterpret_cast<uint32_t *>(layout.ranges), sizeof(layout.ranges));
	layout.push_constant_layout_hash = h.get();

	program.set_pipeline_layout(request_pipeline_layout(layout));

	if (program.get_shader(ShaderStage::Compute))
	{
		auto &shader = *program.get_shader(ShaderStage::Compute);
		VkComputePipelineCreateInfo info = { VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO };
		info.layout = program.get_pipeline_layout()->get_layout();
		info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
		info.stage.module = shader.get_module();
		info.stage.pName = "main";
		info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;

#ifdef GRANITE_SPIRV_DUMP
		LOGI("Compiling SPIR-V file: (%s) %s\n",
		     Shader::stage_to_name(ShaderStage::Compute),
		     (to_string(shader.get_hash()) + ".spv").c_str());
#endif

		VkPipeline compute_pipeline;
		unsigned pipe_index = get_state_recorder().register_compute_pipeline(program.get_hash(), info);
		LOGI("Creating compute pipeline.\n");
		if (vkCreateComputePipelines(device, pipeline_cache, 1, &info, nullptr, &compute_pipeline) != VK_SUCCESS)
			LOGE("Failed to create compute pipeline!\n");
		get_state_recorder().set_compute_pipeline_handle(pipe_index, compute_pipeline);
		program.set_compute_pipeline(compute_pipeline);
	}
}

void Device::init_pipeline_cache()
{
	auto file = Filesystem::get().open("cache://pipeline_cache.bin", FileMode::ReadOnly);
	VkPipelineCacheCreateInfo info = { VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO };

	if (file)
	{
		auto size = file->get_size();
		static const auto uuid_size = sizeof(gpu_props.pipelineCacheUUID);
		if (size >= uuid_size)
		{
			uint8_t *mapped = static_cast<uint8_t *>(file->map());
			if (mapped)
			{
				if (memcmp(gpu_props.pipelineCacheUUID, mapped, uuid_size) == 0)
				{
					info.initialDataSize = size - uuid_size;
					info.pInitialData = mapped + uuid_size;
				}
			}
		}
	}

	vkCreatePipelineCache(device, &info, nullptr, &pipeline_cache);
}

void Device::init_pipeline_state()
{
	auto file = Filesystem::get().open("cache://pipelines.json", FileMode::ReadOnly);
	if (!file)
		return;

	void *mapped = file->map();
	if (!mapped)
	{
		LOGE("Failed to map pipelines.json.\n");
		return;
	}

	try
	{
		LOGI("Replaying cached state.\n");
		Fossilize::StateReplayer replayer;
		replayer.parse(*this, static_cast<const char *>(mapped), file->get_size());
		LOGI("Completed replaying cached state.\n");
		replayer_state = {};
	}
	catch (const exception &e)
	{
		LOGE("Exception caught while parsing pipeline state: %s.\n", e.what());
	}
}

void Device::flush_pipeline_state()
{
	auto serial = state_recorder.serialize();
	auto file = Filesystem::get().open("cache://pipelines.json", FileMode::WriteOnly);
	if (file)
	{
		uint8_t *data = static_cast<uint8_t *>(file->map_write(serial.size()));
		if (data)
		{
			memcpy(data, serial.data(), serial.size());
			file->unmap();
		}
		else
			LOGE("Failed to serialize pipeline data.\n");
	}
}

void Device::flush_pipeline_cache()
{
	static const auto uuid_size = sizeof(gpu_props.pipelineCacheUUID);
	size_t size = 0;
	if (vkGetPipelineCacheData(device, pipeline_cache, &size, nullptr) != VK_SUCCESS)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}

	auto file = Filesystem::get().open("cache://pipeline_cache.bin", FileMode::WriteOnly);
	if (!file)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}

	uint8_t *data = static_cast<uint8_t *>(file->map_write(uuid_size + size));
	if (!data)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}

	memcpy(data, gpu_props.pipelineCacheUUID, uuid_size);
	if (vkGetPipelineCacheData(device, pipeline_cache, &size, data + uuid_size) != VK_SUCCESS)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}
}

void Device::set_context(const Context &context)
{
	instance = context.get_instance();
	gpu = context.get_gpu();
	device = context.get_device();

	graphics_queue_family_index = context.get_graphics_queue_family();
	graphics_queue = context.get_graphics_queue();
	compute_queue_family_index = context.get_compute_queue_family();
	compute_queue = context.get_compute_queue();
	transfer_queue_family_index = context.get_transfer_queue_family();
	transfer_queue = context.get_transfer_queue();

	mem_props = context.get_mem_props();
	gpu_props = context.get_gpu_props();

	init_stock_samplers();
	init_pipeline_cache();
	init_pipeline_state();

	ext = context.get_enabled_device_features();

	managers.memory.init(gpu, device);
	managers.memory.set_supports_dedicated_allocation(ext.supports_dedicated);
	managers.semaphore.init(device);
	managers.fence.init(device);
	managers.event.init(device);
	managers.vbo.init(this, 4 * 1024, 16, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT);
	managers.ibo.init(this, 4 * 1024, 16, VK_BUFFER_USAGE_INDEX_BUFFER_BIT);
	managers.ubo.init(this, 256 * 1024, std::max<VkDeviceSize>(16u, gpu_props.limits.minUniformBufferOffsetAlignment), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT);
	managers.staging.init(this, 64 * 1024, std::max<VkDeviceSize>(16u, gpu_props.limits.optimalBufferCopyOffsetAlignment), VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
}

void Device::init_stock_samplers()
{
	SamplerCreateInfo info = {};
	info.maxLod = VK_LOD_CLAMP_NONE;
	info.maxAnisotropy = 1.0f;

	for (unsigned i = 0; i < static_cast<unsigned>(StockSampler::Count); i++)
	{
		auto mode = static_cast<StockSampler>(i);

		switch (mode)
		{
		case StockSampler::NearestShadow:
		case StockSampler::LinearShadow:
			info.compareEnable = true;
			info.compareOp = VK_COMPARE_OP_LESS_OR_EQUAL;
			break;

		default:
			info.compareEnable = false;
			break;
		}

		switch (mode)
		{
		case StockSampler::TrilinearClamp:
		case StockSampler::TrilinearWrap:
			info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
			break;

		default:
			info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
			break;
		}

		switch (mode)
		{
		case StockSampler::LinearClamp:
		case StockSampler::LinearWrap:
		case StockSampler::TrilinearClamp:
		case StockSampler::TrilinearWrap:
		case StockSampler::LinearShadow:
			info.magFilter = VK_FILTER_LINEAR;
			info.minFilter = VK_FILTER_LINEAR;
			break;

		default:
			info.magFilter = VK_FILTER_NEAREST;
			info.minFilter = VK_FILTER_NEAREST;
			break;
		}

		switch (mode)
		{
		default:
		case StockSampler::LinearWrap:
		case StockSampler::NearestWrap:
		case StockSampler::TrilinearWrap:
			info.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT;
			info.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT;
			info.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT;
			break;

		case StockSampler::LinearClamp:
		case StockSampler::NearestClamp:
		case StockSampler::TrilinearClamp:
		case StockSampler::NearestShadow:
		case StockSampler::LinearShadow:
			info.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
			info.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
			info.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
			break;
		}
		samplers[i] = create_sampler(info);
		samplers[i]->set_internal_sync_object();
	}
}

static void request_block(Device &device, BufferBlock &block, VkDeviceSize size,
                          BufferPool &pool, std::vector<BufferBlock> *dma, std::vector<BufferBlock> &recycle)
{
	if (block.mapped)
		device.unmap_host_buffer(*block.cpu);

	if (block.offset == 0)
	{
		if (block.size == pool.get_block_size())
			pool.recycle_block(move(block));
	}
	else
	{
		if (block.cpu != block.gpu)
		{
			VK_ASSERT(dma);
			dma->push_back(block);
		}

		if (block.size == pool.get_block_size())
			recycle.push_back(block);
	}

	if (size)
		block = pool.request_block(size);
	else
		block = {};
}

void Device::request_vertex_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_vertex_block_nolock(block, size);
}

void Device::request_vertex_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.vbo, &dma.vbo, frame().vbo_blocks);
}

void Device::request_index_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_index_block_nolock(block, size);
}

void Device::request_index_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.ibo, &dma.ibo, frame().ibo_blocks);
}

void Device::request_uniform_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_uniform_block_nolock(block, size);
}

void Device::request_uniform_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.ubo, &dma.ubo, frame().ubo_blocks);
}

void Device::request_staging_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_staging_block_nolock(block, size);
}

void Device::request_staging_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.staging, nullptr, frame().staging_blocks);
}

void Device::submit(CommandBufferHandle cmd, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	LOCK();
	submit_nolock(move(cmd), fence, semaphore, semaphore_alt);
}

void Device::submit_nolock(CommandBufferHandle cmd, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	auto type = cmd->get_command_buffer_type();
	auto &pool = get_command_pool(type, cmd->get_thread_index());
	auto &submissions = get_queue_submissions(type);

	pool.signal_submitted(cmd->get_command_buffer());
	cmd->end();
	submissions.push_back(move(cmd));

	VkFence cleared_fence = VK_NULL_HANDLE;

	if (fence || semaphore || semaphore_alt)
		submit_queue(type, fence ? &cleared_fence : nullptr, semaphore, semaphore_alt);

	if (fence)
	{
		VK_ASSERT(!*fence);
		*fence = make_handle<FenceHolder>(this, cleared_fence);
	}

	decrement_frame_counter_nolock();
}

void Device::submit_empty(CommandBuffer::Type type, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	LOCK();
	submit_empty_nolock(type, fence, semaphore, semaphore_alt);
}

void Device::submit_empty_nolock(CommandBuffer::Type type, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	if (type != CommandBuffer::Type::Transfer)
		flush_frame(CommandBuffer::Type::Transfer);

	VkFence cleared_fence = VK_NULL_HANDLE;
	submit_queue(type, fence ? &cleared_fence : nullptr, semaphore, semaphore_alt);
	if (fence)
		*fence = make_handle<FenceHolder>(this, cleared_fence);
}

void Device::submit_empty_inner(CommandBuffer::Type type, VkFence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	auto &data = get_queue_data(type);
	VkSubmitInfo submit = { VK_STRUCTURE_TYPE_SUBMIT_INFO };

	// Add external wait semaphores.
	vector<VkSemaphore> waits;
	vector<VkSemaphore> signals;
	auto stages = move(data.wait_stages);

	for (auto &semaphore : data.wait_semaphores)
	{
		auto wait = semaphore->consume();
		if (semaphore->can_recycle())
			frame().recycled_semaphores.push_back(wait);
		else
			frame().destroyed_semaphores.push_back(wait);
		waits.push_back(wait);
	}
	data.wait_stages.clear();
	data.wait_semaphores.clear();

	// Add external signal semaphores.
	VkSemaphore cleared_semaphore = VK_NULL_HANDLE;
	VkSemaphore cleared_semaphore_alt = VK_NULL_HANDLE;
	if (semaphore)
	{
		cleared_semaphore = managers.semaphore.request_cleared_semaphore();
		signals.push_back(cleared_semaphore);
	}

	if (semaphore_alt)
	{
		cleared_semaphore_alt = managers.semaphore.request_cleared_semaphore();
		signals.push_back(cleared_semaphore_alt);
	}

	submit.signalSemaphoreCount = signals.size();
	submit.waitSemaphoreCount = waits.size();
	if (!signals.empty())
		submit.pSignalSemaphores = signals.data();
	if (!stages.empty())
		submit.pWaitDstStageMask = stages.data();
	if (!waits.empty())
		submit.pWaitSemaphores = waits.data();

	VkQueue queue;
	switch (type)
	{
	default:
	case CommandBuffer::Type::Graphics:
		queue = graphics_queue;
		break;
	case CommandBuffer::Type::Compute:
		queue = compute_queue;
		break;
	case CommandBuffer::Type::Transfer:
		queue = transfer_queue;
		break;
	}

	VkFence cleared_fence = fence ? managers.fence.request_cleared_fence() : VK_NULL_HANDLE;
	if (queue_lock_callback)
		queue_lock_callback();
#if defined(VULKAN_DEBUG) && defined(SUBMIT_DEBUG)
	if (cleared_fence)
		LOGI("Signalling Fence: %llx\n", reinterpret_cast<unsigned long long>(cleared_fence));
#endif
	VkResult result = vkQueueSubmit(queue, 1, &submit, cleared_fence);
	if (ImplementationQuirks::get().queue_wait_on_submission)
		vkQueueWaitIdle(queue);
	if (queue_unlock_callback)
		queue_unlock_callback();

	if (result != VK_SUCCESS)
		LOGE("vkQueueSubmit failed (code: %d).\n", int(result));

	if (fence)
	{
		frame().wait_fences.push_back(cleared_fence);
		*fence = cleared_fence;
		data.need_fence = false;
	}
	else
		data.need_fence = true;

	if (semaphore)
	{
		VK_ASSERT(!*semaphore);
		*semaphore = make_handle<SemaphoreHolder>(this, cleared_semaphore, true);
	}

	if (semaphore_alt)
	{
		VK_ASSERT(!*semaphore_alt);
		*semaphore_alt = make_handle<SemaphoreHolder>(this, cleared_semaphore_alt, true);
	}

#if defined(VULKAN_DEBUG) && defined(SUBMIT_DEBUG)
	const char *queue_name = nullptr;
	switch (type)
	{
	default:
	case CommandBuffer::Type::Graphics:
		queue_name = "Graphics";
		break;
	case CommandBuffer::Type::Compute:
		queue_name = "Compute";
		break;
	case CommandBuffer::Type::Transfer:
		queue_name = "Transfer";
		break;
	}

	LOGI("Empty submission to %s queue:\n", queue_name);
	for (uint32_t i = 0; i < submit.waitSemaphoreCount; i++)
	{
		LOGI("  Waiting for semaphore: %llx in stages %s\n",
		     reinterpret_cast<unsigned long long>(submit.pWaitSemaphores[i]),
		     stage_flags_to_string(submit.pWaitDstStageMask[i]).c_str());
	}

	for (uint32_t i = 0; i < submit.signalSemaphoreCount; i++)
	{
		LOGI("  Signalling semaphore: %llx\n",
		     reinterpret_cast<unsigned long long>(submit.pSignalSemaphores[i]));
	}
#endif
}

void Device::submit_staging(CommandBufferHandle cmd, VkBufferUsageFlags usage, bool flush)
{
	auto access = buffer_usage_to_possible_access(usage);
	auto stages = buffer_usage_to_possible_stages(usage);

	if (transfer_queue == graphics_queue && transfer_queue == compute_queue)
	{
		// For single-queue systems, just use a pipeline barrier.
		cmd->barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, stages, access);
		submit_nolock(cmd);
	}
	else
	{
		auto compute_stages = stages &
		                      (VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
		                       VK_PIPELINE_STAGE_TRANSFER_BIT);

		auto compute_access = access &
		                      (VK_ACCESS_SHADER_READ_BIT |
		                       VK_ACCESS_SHADER_WRITE_BIT |
		                       VK_ACCESS_TRANSFER_READ_BIT |
		                       VK_ACCESS_UNIFORM_READ_BIT |
		                       VK_ACCESS_TRANSFER_WRITE_BIT);

		auto graphics_stages = stages;

		if (transfer_queue == graphics_queue)
		{
			cmd->barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
			             graphics_stages, access);

			if (compute_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Compute, sem, compute_stages, flush);
			}
			else
				submit_nolock(cmd);
		}
		else if (transfer_queue == compute_queue)
		{
			cmd->barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
			             compute_stages, compute_access);

			if (graphics_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Graphics, sem, graphics_stages, flush);
			}
			else
				submit_nolock(cmd);
		}
		else
		{
			if (graphics_stages != 0 && compute_stages != 0)
			{
				Semaphore sem_graphics, sem_compute;
				submit_nolock(cmd, nullptr, &sem_graphics, &sem_compute);
				add_wait_semaphore_nolock(CommandBuffer::Type::Graphics, sem_graphics, graphics_stages, flush);
				add_wait_semaphore_nolock(CommandBuffer::Type::Compute, sem_compute, compute_stages, flush);
			}
			else if (graphics_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Graphics, sem, graphics_stages, flush);
			}
			else if (compute_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Compute, sem, compute_stages, flush);
			}
			else
				submit_nolock(cmd);
		}
	}
}

void Device::submit_queue(CommandBuffer::Type type, VkFence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	// Always check if we need to flush pending transfers.
	if (type != CommandBuffer::Type::Transfer)
		flush_frame(CommandBuffer::Type::Transfer);

	auto &data = get_queue_data(type);
	auto &submissions = get_queue_submissions(type);

	if (submissions.empty())
	{
		if (fence || semaphore || semaphore_alt)
			submit_empty_inner(type, fence, semaphore, semaphore_alt);
		return;
	}

	vector<VkCommandBuffer> cmds;
	cmds.reserve(submissions.size());

	vector<VkSubmitInfo> submits;
	submits.reserve(2);
	size_t last_cmd = 0;<commit_msg>Deal with secondary graphics in submit.<commit_after>/* Copyright (c) 2017-2018 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include "device.hpp"
#include "format.hpp"
#include "thread_group.hpp"
#include "type_to_string.hpp"
#include "quirks.hpp"
#include "enum_cast.hpp"
#include <algorithm>
#include <string.h>

#ifdef VULKAN_MT
#define LOCK() std::lock_guard<std::mutex> holder__{lock.lock}
#define DRAIN_FRAME_LOCK() \
	std::unique_lock<std::mutex> holder__{lock.lock}; \
	lock.cond.wait(holder__, [&]() { \
		return lock.counter == 0; \
	})
#else
#define LOCK() ((void)0)
#define DRAIN_FRAME_LOCK() VK_ASSERT(lock.counter == 0)
#endif

using namespace std;
using namespace Util;

namespace Vulkan
{
Device::Device()
    : framebuffer_allocator(this)
    , transient_allocator(this)
	, physical_allocator(this)
	, shader_manager(this)
	, texture_manager(this)
{
	cookie.store(0);

	ThreadGroup::get_global();
	ThreadGroup::register_main_thread();
}

Semaphore Device::request_semaphore()
{
	LOCK();
	auto semaphore = managers.semaphore.request_cleared_semaphore();
	auto ptr = make_handle<SemaphoreHolder>(this, semaphore, false);
	return ptr;
}

#ifndef _WIN32
Semaphore Device::request_imported_semaphore(int fd, VkExternalSemaphoreHandleTypeFlagBitsKHR handle_type)
{
	LOCK();
	if (!ext.supports_external)
		return {};

	VkExternalSemaphorePropertiesKHR props = { VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES_KHR };
	VkPhysicalDeviceExternalSemaphoreInfoKHR info = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO_KHR };
	info.handleType = handle_type;

	vkGetPhysicalDeviceExternalSemaphorePropertiesKHR(gpu, &info, &props);
	if ((props.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT_KHR) == 0)
		return Semaphore(nullptr);

	auto semaphore = managers.semaphore.request_cleared_semaphore();

	VkImportSemaphoreFdInfoKHR import = { VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR };
	import.fd = fd;
	import.semaphore = semaphore;
	import.handleType = handle_type;
	import.flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT_KHR;
	auto ptr = make_handle<SemaphoreHolder>(this, semaphore, false);

	if (vkImportSemaphoreFdKHR(device, &import) != VK_SUCCESS)
		return Semaphore(nullptr);

	ptr->signal_external();
	ptr->destroy_on_consume();
	return ptr;
}
#endif

void Device::add_wait_semaphore(CommandBuffer::Type type, Semaphore semaphore, VkPipelineStageFlags stages, bool flush)
{
	LOCK();
	add_wait_semaphore_nolock(type, semaphore, stages, flush);
}

void Device::add_wait_semaphore_nolock(CommandBuffer::Type type, Semaphore semaphore, VkPipelineStageFlags stages,
                                       bool flush)
{
	VK_ASSERT(stages != 0);
	if (flush)
		flush_frame(type);
	auto &data = get_queue_data(type);

#ifdef VULKAN_DEBUG
	for (auto &sem : data.wait_semaphores)
		VK_ASSERT(sem.get() != semaphore.get());
#endif

	semaphore->signal_pending_wait();
	data.wait_semaphores.push_back(semaphore);
	data.wait_stages.push_back(stages);
	data.need_fence = true;

	// Sanity check.
	VK_ASSERT(data.wait_semaphores.size() < 1024);
}

void *Device::map_host_buffer(Buffer &buffer, MemoryAccessFlags access)
{
	void *host = managers.memory.map_memory(&buffer.get_allocation(), access);
	return host;
}

void Device::unmap_host_buffer(const Buffer &buffer)
{
	managers.memory.unmap_memory(buffer.get_allocation());
}

Shader *Device::request_shader(const uint32_t *data, size_t size)
{
	Util::Hasher hasher;
	hasher.data(data, size);

	auto hash = hasher.get();
	auto *ret = shaders.find(hash);
	if (!ret)
		ret = shaders.insert(hash, make_unique<Shader>(hash, this, data, size));
	return ret;
}

bool Device::enqueue_create_shader_module(Fossilize::Hash hash, unsigned, const VkShaderModuleCreateInfo *create_info, VkShaderModule *module)
{
	auto *ret = shaders.insert(hash, make_unique<Shader>(hash, this, create_info->pCode, create_info->codeSize));
	*module = ret->get_module();
	replayer_state.shader_map[*module] = ret;
	return true;
}

bool Device::enqueue_create_compute_pipeline(Fossilize::Hash hash, unsigned,
                                             const VkComputePipelineCreateInfo *create_info, VkPipeline *pipeline)
{
	// Find the Shader* associated with this VkShaderModule and just use that.
	auto itr = replayer_state.shader_map.find(create_info->stage.module);
	if (itr == end(replayer_state.shader_map))
		return false;

	auto *ret = request_program(itr->second);
	*pipeline = ret->get_compute_pipeline();
	return true;
}

Program *Device::request_program(Vulkan::Shader *compute)
{
	Util::Hasher hasher;
	hasher.u64(compute->get_hash());

	auto hash = hasher.get();
	auto *ret = programs.find(hash);
	if (!ret)
		ret = programs.insert(hash, make_unique<Program>(hash, this, compute));
	return ret;
}

Program *Device::request_program(const uint32_t *compute_data, size_t compute_size)
{
	auto *compute = request_shader(compute_data, compute_size);
	return request_program(compute);
}

bool Device::enqueue_create_graphics_pipeline(Fossilize::Hash hash, unsigned,
                                              const VkGraphicsPipelineCreateInfo *create_info, VkPipeline *pipeline)
{
	auto info = *create_info;

	if (info.stageCount != 2)
		return false;
	if (info.pStages[0].stage != VK_SHADER_STAGE_VERTEX_BIT)
		return false;
	if (info.pStages[1].stage != VK_SHADER_STAGE_FRAGMENT_BIT)
		return false;

	// Find the Shader* associated with this VkShaderModule and just use that.
	auto vertex_itr = replayer_state.shader_map.find(info.pStages[0].module);
	if (vertex_itr == end(replayer_state.shader_map))
		return false;

	// Find the Shader* associated with this VkShaderModule and just use that.
	auto fragment_itr = replayer_state.shader_map.find(info.pStages[1].module);
	if (fragment_itr == end(replayer_state.shader_map))
		return false;

	auto *ret = request_program(vertex_itr->second, fragment_itr->second);

	// The layout is dummy, resolve it here.
	info.layout = ret->get_pipeline_layout()->get_layout();

	get_state_recorder().register_graphics_pipeline(hash, info);
	LOGI("Creating graphics pipeline.\n");
	VkResult res = vkCreateGraphicsPipelines(device, pipeline_cache, 1, &info, nullptr, pipeline);
	if (res != VK_SUCCESS)
		LOGE("Failed to create graphics pipeline!\n");
	*pipeline = ret->add_graphics_pipeline(hash, *pipeline);
	return true;
}

Program *Device::request_program(Vulkan::Shader *vertex, Vulkan::Shader *fragment)
{
	Util::Hasher hasher;
	hasher.u64(vertex->get_hash());
	hasher.u64(fragment->get_hash());

	auto hash = hasher.get();
	auto *ret = programs.find(hash);

	if (!ret)
		ret = programs.insert(hash, make_unique<Program>(hash, this, vertex, fragment));
	return ret;
}

Program *Device::request_program(const uint32_t *vertex_data, size_t vertex_size, const uint32_t *fragment_data,
                                 size_t fragment_size)
{
	auto *vertex = request_shader(vertex_data, vertex_size);
	auto *fragment = request_shader(fragment_data, fragment_size);
	return request_program(vertex, fragment);
}

PipelineLayout *Device::request_pipeline_layout(const CombinedResourceLayout &layout)
{
	Hasher h;
	h.data(reinterpret_cast<const uint32_t *>(layout.sets), sizeof(layout.sets));
	h.data(&layout.stages_for_bindings[0][0], sizeof(layout.stages_for_bindings));
	h.data(reinterpret_cast<const uint32_t *>(layout.ranges), sizeof(layout.ranges));
	h.u32(layout.attribute_mask);
	h.u32(layout.render_target_mask);

	auto hash = h.get();
	auto *ret = pipeline_layouts.find(hash);
	if (!ret)
		ret = pipeline_layouts.insert(hash, make_unique<PipelineLayout>(hash, this, layout));
	return ret;
}

DescriptorSetAllocator *Device::request_descriptor_set_allocator(const DescriptorSetLayout &layout, const uint32_t *stages_for_bindings)
{
	Hasher h;
	h.data(reinterpret_cast<const uint32_t *>(&layout), sizeof(layout));
	h.data(stages_for_bindings, sizeof(uint32_t) * VULKAN_NUM_BINDINGS);
	auto hash = h.get();

	auto *ret = descriptor_set_allocators.find(hash);
	if (!ret)
		ret = descriptor_set_allocators.insert(hash, make_unique<DescriptorSetAllocator>(hash, this, layout, stages_for_bindings));
	return ret;
}

void Device::bake_program(Program &program)
{
	CombinedResourceLayout layout;
	if (program.get_shader(ShaderStage::Vertex))
		layout.attribute_mask = program.get_shader(ShaderStage::Vertex)->get_layout().input_mask;
	if (program.get_shader(ShaderStage::Fragment))
		layout.render_target_mask = program.get_shader(ShaderStage::Fragment)->get_layout().output_mask;

	layout.descriptor_set_mask = 0;

	for (unsigned i = 0; i < static_cast<unsigned>(ShaderStage::Count); i++)
	{
		auto *shader = program.get_shader(static_cast<ShaderStage>(i));
		if (!shader)
			continue;

		uint32_t stage_mask = 1u << i;

		auto &shader_layout = shader->get_layout();
		for (unsigned set = 0; set < VULKAN_NUM_DESCRIPTOR_SETS; set++)
		{
			layout.sets[set].sampled_image_mask |= shader_layout.sets[set].sampled_image_mask;
			layout.sets[set].storage_image_mask |= shader_layout.sets[set].storage_image_mask;
			layout.sets[set].uniform_buffer_mask |= shader_layout.sets[set].uniform_buffer_mask;
			layout.sets[set].storage_buffer_mask |= shader_layout.sets[set].storage_buffer_mask;
			layout.sets[set].sampled_buffer_mask |= shader_layout.sets[set].sampled_buffer_mask;
			layout.sets[set].input_attachment_mask |= shader_layout.sets[set].input_attachment_mask;
			layout.sets[set].sampler_mask |= shader_layout.sets[set].sampler_mask;
			layout.sets[set].separate_image_mask |= shader_layout.sets[set].separate_image_mask;
			layout.sets[set].fp_mask |= shader_layout.sets[set].fp_mask;

			uint32_t active_binds =
					shader_layout.sets[set].sampled_image_mask |
					shader_layout.sets[set].storage_image_mask |
					shader_layout.sets[set].uniform_buffer_mask|
					shader_layout.sets[set].storage_buffer_mask |
					shader_layout.sets[set].sampled_buffer_mask |
					shader_layout.sets[set].input_attachment_mask |
					shader_layout.sets[set].sampler_mask |
					shader_layout.sets[set].separate_image_mask;

			if (active_binds)
				layout.stages_for_sets[set] |= stage_mask;

			for_each_bit(active_binds, [&](uint32_t bit) {
				layout.stages_for_bindings[set][bit] |= stage_mask;
			});
		}

		layout.ranges[i].stageFlags = 1u << i;
		layout.ranges[i].offset = shader_layout.push_constant_offset;
		layout.ranges[i].size = shader_layout.push_constant_range;
	}

	for (unsigned i = 0; i < VULKAN_NUM_DESCRIPTOR_SETS; i++)
	{
		if (layout.stages_for_sets[i] != 0)
			layout.descriptor_set_mask |= 1u << i;
	}

	Hasher h;
	h.data(reinterpret_cast<uint32_t *>(layout.ranges), sizeof(layout.ranges));
	layout.push_constant_layout_hash = h.get();

	program.set_pipeline_layout(request_pipeline_layout(layout));

	if (program.get_shader(ShaderStage::Compute))
	{
		auto &shader = *program.get_shader(ShaderStage::Compute);
		VkComputePipelineCreateInfo info = { VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO };
		info.layout = program.get_pipeline_layout()->get_layout();
		info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
		info.stage.module = shader.get_module();
		info.stage.pName = "main";
		info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;

#ifdef GRANITE_SPIRV_DUMP
		LOGI("Compiling SPIR-V file: (%s) %s\n",
		     Shader::stage_to_name(ShaderStage::Compute),
		     (to_string(shader.get_hash()) + ".spv").c_str());
#endif

		VkPipeline compute_pipeline;
		unsigned pipe_index = get_state_recorder().register_compute_pipeline(program.get_hash(), info);
		LOGI("Creating compute pipeline.\n");
		if (vkCreateComputePipelines(device, pipeline_cache, 1, &info, nullptr, &compute_pipeline) != VK_SUCCESS)
			LOGE("Failed to create compute pipeline!\n");
		get_state_recorder().set_compute_pipeline_handle(pipe_index, compute_pipeline);
		program.set_compute_pipeline(compute_pipeline);
	}
}

void Device::init_pipeline_cache()
{
	auto file = Filesystem::get().open("cache://pipeline_cache.bin", FileMode::ReadOnly);
	VkPipelineCacheCreateInfo info = { VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO };

	if (file)
	{
		auto size = file->get_size();
		static const auto uuid_size = sizeof(gpu_props.pipelineCacheUUID);
		if (size >= uuid_size)
		{
			uint8_t *mapped = static_cast<uint8_t *>(file->map());
			if (mapped)
			{
				if (memcmp(gpu_props.pipelineCacheUUID, mapped, uuid_size) == 0)
				{
					info.initialDataSize = size - uuid_size;
					info.pInitialData = mapped + uuid_size;
				}
			}
		}
	}

	vkCreatePipelineCache(device, &info, nullptr, &pipeline_cache);
}

void Device::init_pipeline_state()
{
	auto file = Filesystem::get().open("cache://pipelines.json", FileMode::ReadOnly);
	if (!file)
		return;

	void *mapped = file->map();
	if (!mapped)
	{
		LOGE("Failed to map pipelines.json.\n");
		return;
	}

	try
	{
		LOGI("Replaying cached state.\n");
		Fossilize::StateReplayer replayer;
		replayer.parse(*this, static_cast<const char *>(mapped), file->get_size());
		LOGI("Completed replaying cached state.\n");
		replayer_state = {};
	}
	catch (const exception &e)
	{
		LOGE("Exception caught while parsing pipeline state: %s.\n", e.what());
	}
}

void Device::flush_pipeline_state()
{
	auto serial = state_recorder.serialize();
	auto file = Filesystem::get().open("cache://pipelines.json", FileMode::WriteOnly);
	if (file)
	{
		uint8_t *data = static_cast<uint8_t *>(file->map_write(serial.size()));
		if (data)
		{
			memcpy(data, serial.data(), serial.size());
			file->unmap();
		}
		else
			LOGE("Failed to serialize pipeline data.\n");
	}
}

void Device::flush_pipeline_cache()
{
	static const auto uuid_size = sizeof(gpu_props.pipelineCacheUUID);
	size_t size = 0;
	if (vkGetPipelineCacheData(device, pipeline_cache, &size, nullptr) != VK_SUCCESS)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}

	auto file = Filesystem::get().open("cache://pipeline_cache.bin", FileMode::WriteOnly);
	if (!file)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}

	uint8_t *data = static_cast<uint8_t *>(file->map_write(uuid_size + size));
	if (!data)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}

	memcpy(data, gpu_props.pipelineCacheUUID, uuid_size);
	if (vkGetPipelineCacheData(device, pipeline_cache, &size, data + uuid_size) != VK_SUCCESS)
	{
		LOGE("Failed to get pipeline cache data.\n");
		return;
	}
}

void Device::set_context(const Context &context)
{
	instance = context.get_instance();
	gpu = context.get_gpu();
	device = context.get_device();

	graphics_queue_family_index = context.get_graphics_queue_family();
	graphics_queue = context.get_graphics_queue();
	compute_queue_family_index = context.get_compute_queue_family();
	compute_queue = context.get_compute_queue();
	transfer_queue_family_index = context.get_transfer_queue_family();
	transfer_queue = context.get_transfer_queue();

	mem_props = context.get_mem_props();
	gpu_props = context.get_gpu_props();

	init_stock_samplers();
	init_pipeline_cache();
	init_pipeline_state();

	ext = context.get_enabled_device_features();

	managers.memory.init(gpu, device);
	managers.memory.set_supports_dedicated_allocation(ext.supports_dedicated);
	managers.semaphore.init(device);
	managers.fence.init(device);
	managers.event.init(device);
	managers.vbo.init(this, 4 * 1024, 16, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT);
	managers.ibo.init(this, 4 * 1024, 16, VK_BUFFER_USAGE_INDEX_BUFFER_BIT);
	managers.ubo.init(this, 256 * 1024, std::max<VkDeviceSize>(16u, gpu_props.limits.minUniformBufferOffsetAlignment), VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT);
	managers.staging.init(this, 64 * 1024, std::max<VkDeviceSize>(16u, gpu_props.limits.optimalBufferCopyOffsetAlignment), VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
}

void Device::init_stock_samplers()
{
	SamplerCreateInfo info = {};
	info.maxLod = VK_LOD_CLAMP_NONE;
	info.maxAnisotropy = 1.0f;

	for (unsigned i = 0; i < static_cast<unsigned>(StockSampler::Count); i++)
	{
		auto mode = static_cast<StockSampler>(i);

		switch (mode)
		{
		case StockSampler::NearestShadow:
		case StockSampler::LinearShadow:
			info.compareEnable = true;
			info.compareOp = VK_COMPARE_OP_LESS_OR_EQUAL;
			break;

		default:
			info.compareEnable = false;
			break;
		}

		switch (mode)
		{
		case StockSampler::TrilinearClamp:
		case StockSampler::TrilinearWrap:
			info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
			break;

		default:
			info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
			break;
		}

		switch (mode)
		{
		case StockSampler::LinearClamp:
		case StockSampler::LinearWrap:
		case StockSampler::TrilinearClamp:
		case StockSampler::TrilinearWrap:
		case StockSampler::LinearShadow:
			info.magFilter = VK_FILTER_LINEAR;
			info.minFilter = VK_FILTER_LINEAR;
			break;

		default:
			info.magFilter = VK_FILTER_NEAREST;
			info.minFilter = VK_FILTER_NEAREST;
			break;
		}

		switch (mode)
		{
		default:
		case StockSampler::LinearWrap:
		case StockSampler::NearestWrap:
		case StockSampler::TrilinearWrap:
			info.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT;
			info.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT;
			info.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT;
			break;

		case StockSampler::LinearClamp:
		case StockSampler::NearestClamp:
		case StockSampler::TrilinearClamp:
		case StockSampler::NearestShadow:
		case StockSampler::LinearShadow:
			info.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
			info.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
			info.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
			break;
		}
		samplers[i] = create_sampler(info);
		samplers[i]->set_internal_sync_object();
	}
}

static void request_block(Device &device, BufferBlock &block, VkDeviceSize size,
                          BufferPool &pool, std::vector<BufferBlock> *dma, std::vector<BufferBlock> &recycle)
{
	if (block.mapped)
		device.unmap_host_buffer(*block.cpu);

	if (block.offset == 0)
	{
		if (block.size == pool.get_block_size())
			pool.recycle_block(move(block));
	}
	else
	{
		if (block.cpu != block.gpu)
		{
			VK_ASSERT(dma);
			dma->push_back(block);
		}

		if (block.size == pool.get_block_size())
			recycle.push_back(block);
	}

	if (size)
		block = pool.request_block(size);
	else
		block = {};
}

void Device::request_vertex_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_vertex_block_nolock(block, size);
}

void Device::request_vertex_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.vbo, &dma.vbo, frame().vbo_blocks);
}

void Device::request_index_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_index_block_nolock(block, size);
}

void Device::request_index_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.ibo, &dma.ibo, frame().ibo_blocks);
}

void Device::request_uniform_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_uniform_block_nolock(block, size);
}

void Device::request_uniform_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.ubo, &dma.ubo, frame().ubo_blocks);
}

void Device::request_staging_block(BufferBlock &block, VkDeviceSize size)
{
	LOCK();
	request_staging_block_nolock(block, size);
}

void Device::request_staging_block_nolock(BufferBlock &block, VkDeviceSize size)
{
	request_block(*this, block, size, managers.staging, nullptr, frame().staging_blocks);
}

void Device::submit(CommandBufferHandle cmd, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	LOCK();
	submit_nolock(move(cmd), fence, semaphore, semaphore_alt);
}

void Device::submit_nolock(CommandBufferHandle cmd, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	auto type = cmd->get_command_buffer_type();
	auto &pool = get_command_pool(type, cmd->get_thread_index());
	auto &submissions = get_queue_submissions(type);

	pool.signal_submitted(cmd->get_command_buffer());
	cmd->end();
	submissions.push_back(move(cmd));

	VkFence cleared_fence = VK_NULL_HANDLE;

	if (fence || semaphore || semaphore_alt)
		submit_queue(type, fence ? &cleared_fence : nullptr, semaphore, semaphore_alt);

	if (fence)
	{
		VK_ASSERT(!*fence);
		*fence = make_handle<FenceHolder>(this, cleared_fence);
	}

	decrement_frame_counter_nolock();
}

void Device::submit_empty(CommandBuffer::Type type, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	LOCK();
	submit_empty_nolock(type, fence, semaphore, semaphore_alt);
}

void Device::submit_empty_nolock(CommandBuffer::Type type, Fence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	if (type != CommandBuffer::Type::Transfer)
		flush_frame(CommandBuffer::Type::Transfer);

	VkFence cleared_fence = VK_NULL_HANDLE;
	submit_queue(type, fence ? &cleared_fence : nullptr, semaphore, semaphore_alt);
	if (fence)
		*fence = make_handle<FenceHolder>(this, cleared_fence);
}

void Device::submit_empty_inner(CommandBuffer::Type type, VkFence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	auto &data = get_queue_data(type);
	VkSubmitInfo submit = { VK_STRUCTURE_TYPE_SUBMIT_INFO };

	// Add external wait semaphores.
	vector<VkSemaphore> waits;
	vector<VkSemaphore> signals;
	auto stages = move(data.wait_stages);

	for (auto &semaphore : data.wait_semaphores)
	{
		auto wait = semaphore->consume();
		if (semaphore->can_recycle())
			frame().recycled_semaphores.push_back(wait);
		else
			frame().destroyed_semaphores.push_back(wait);
		waits.push_back(wait);
	}
	data.wait_stages.clear();
	data.wait_semaphores.clear();

	// Add external signal semaphores.
	VkSemaphore cleared_semaphore = VK_NULL_HANDLE;
	VkSemaphore cleared_semaphore_alt = VK_NULL_HANDLE;
	if (semaphore)
	{
		cleared_semaphore = managers.semaphore.request_cleared_semaphore();
		signals.push_back(cleared_semaphore);
	}

	if (semaphore_alt)
	{
		cleared_semaphore_alt = managers.semaphore.request_cleared_semaphore();
		signals.push_back(cleared_semaphore_alt);
	}

	submit.signalSemaphoreCount = signals.size();
	submit.waitSemaphoreCount = waits.size();
	if (!signals.empty())
		submit.pSignalSemaphores = signals.data();
	if (!stages.empty())
		submit.pWaitDstStageMask = stages.data();
	if (!waits.empty())
		submit.pWaitSemaphores = waits.data();

	VkQueue queue;
	switch (type)
	{
	default:
	case CommandBuffer::Type::Graphics:
		queue = graphics_queue;
		break;
	case CommandBuffer::Type::Compute:
		queue = compute_queue;
		break;
	case CommandBuffer::Type::Transfer:
		queue = transfer_queue;
		break;
	}

	VkFence cleared_fence = fence ? managers.fence.request_cleared_fence() : VK_NULL_HANDLE;
	if (queue_lock_callback)
		queue_lock_callback();
#if defined(VULKAN_DEBUG) && defined(SUBMIT_DEBUG)
	if (cleared_fence)
		LOGI("Signalling Fence: %llx\n", reinterpret_cast<unsigned long long>(cleared_fence));
#endif
	VkResult result = vkQueueSubmit(queue, 1, &submit, cleared_fence);
	if (ImplementationQuirks::get().queue_wait_on_submission)
		vkQueueWaitIdle(queue);
	if (queue_unlock_callback)
		queue_unlock_callback();

	if (result != VK_SUCCESS)
		LOGE("vkQueueSubmit failed (code: %d).\n", int(result));

	if (fence)
	{
		frame().wait_fences.push_back(cleared_fence);
		*fence = cleared_fence;
		data.need_fence = false;
	}
	else
		data.need_fence = true;

	if (semaphore)
	{
		VK_ASSERT(!*semaphore);
		*semaphore = make_handle<SemaphoreHolder>(this, cleared_semaphore, true);
	}

	if (semaphore_alt)
	{
		VK_ASSERT(!*semaphore_alt);
		*semaphore_alt = make_handle<SemaphoreHolder>(this, cleared_semaphore_alt, true);
	}

#if defined(VULKAN_DEBUG) && defined(SUBMIT_DEBUG)
	const char *queue_name = nullptr;
	switch (type)
	{
	default:
	case CommandBuffer::Type::Graphics:
		queue_name = "Graphics";
		break;
	case CommandBuffer::Type::Compute:
		queue_name = "Compute";
		break;
	case CommandBuffer::Type::Transfer:
		queue_name = "Transfer";
		break;
	}

	LOGI("Empty submission to %s queue:\n", queue_name);
	for (uint32_t i = 0; i < submit.waitSemaphoreCount; i++)
	{
		LOGI("  Waiting for semaphore: %llx in stages %s\n",
		     reinterpret_cast<unsigned long long>(submit.pWaitSemaphores[i]),
		     stage_flags_to_string(submit.pWaitDstStageMask[i]).c_str());
	}

	for (uint32_t i = 0; i < submit.signalSemaphoreCount; i++)
	{
		LOGI("  Signalling semaphore: %llx\n",
		     reinterpret_cast<unsigned long long>(submit.pSignalSemaphores[i]));
	}
#endif
}

void Device::submit_staging(CommandBufferHandle cmd, VkBufferUsageFlags usage, bool flush)
{
	auto access = buffer_usage_to_possible_access(usage);
	auto stages = buffer_usage_to_possible_stages(usage);

	if (transfer_queue == graphics_queue && transfer_queue == compute_queue)
	{
		// For single-queue systems, just use a pipeline barrier.
		cmd->barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, stages, access);
		submit_nolock(cmd);
	}
	else
	{
		auto compute_stages = stages &
		                      (VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
		                       VK_PIPELINE_STAGE_TRANSFER_BIT);

		auto compute_access = access &
		                      (VK_ACCESS_SHADER_READ_BIT |
		                       VK_ACCESS_SHADER_WRITE_BIT |
		                       VK_ACCESS_TRANSFER_READ_BIT |
		                       VK_ACCESS_UNIFORM_READ_BIT |
		                       VK_ACCESS_TRANSFER_WRITE_BIT);

		auto graphics_stages = stages;

		if (transfer_queue == graphics_queue)
		{
			cmd->barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
			             graphics_stages, access);

			if (compute_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Compute, sem, compute_stages, flush);
			}
			else
				submit_nolock(cmd);
		}
		else if (transfer_queue == compute_queue)
		{
			cmd->barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
			             compute_stages, compute_access);

			if (graphics_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Graphics, sem, graphics_stages, flush);
			}
			else
				submit_nolock(cmd);
		}
		else
		{
			if (graphics_stages != 0 && compute_stages != 0)
			{
				Semaphore sem_graphics, sem_compute;
				submit_nolock(cmd, nullptr, &sem_graphics, &sem_compute);
				add_wait_semaphore_nolock(CommandBuffer::Type::Graphics, sem_graphics, graphics_stages, flush);
				add_wait_semaphore_nolock(CommandBuffer::Type::Compute, sem_compute, compute_stages, flush);
			}
			else if (graphics_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Graphics, sem, graphics_stages, flush);
			}
			else if (compute_stages != 0)
			{
				Semaphore sem;
				submit_nolock(cmd, nullptr, &sem);
				add_wait_semaphore_nolock(CommandBuffer::Type::Compute, sem, compute_stages, flush);
			}
			else
				submit_nolock(cmd);
		}
	}
}

void Device::submit_queue(CommandBuffer::Type type, VkFence *fence, Semaphore *semaphore, Semaphore *semaphore_alt)
{
	if (type == CommandBuffer::Type::SecondaryGraphics)
	{
		if (graphics_queue_family_index == compute_queue_family_index)
			type = CommandBuffer::Type::Compute;
		else
			type = CommandBuffer::Type::Graphics;
	}

	// Always check if we need to flush pending transfers.
	if (type != CommandBuffer::Type::Transfer)
		flush_frame(CommandBuffer::Type::Transfer);

	auto &data = get_queue_data(type);
	auto &submissions = get_queue_submissions(type);

	if (submissions.empty())
	{
		if (fence || semaphore || semaphore_alt)
			submit_empty_inner(type, fence, semaphore, semaphore_alt);
		return;
	}

	vector<VkCommandBuffer> cmds;
	cmds.reserve(submissions.size());

	vector<VkSubmitInfo> submits;
	submits.reserve(2);
	size_t last_cmd = 0;
{'fraction_non_alphanumeric': 0.12062066434849464, 'fraction_numerical': 0.004578098516486397, 'mean_word_length': 5.437570303712036, 'pattern_counts': {'":': 0, '<': 109, '<?xml version=': 0, '>': 163, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1155437', 'n_tokens_mistral': 21642, 'n_tokens_neox': 20363, 'n_words': 3669}