<commit_before>#include "import_asset_dialog.h"
#include "assimp/DefaultLogger.hpp"
#include "assimp/postprocess.h"
#include "assimp/ProgressHandler.hpp"
#include "assimp/scene.h"
#include "core/crc32.h"
#include "core/FS/ifile.h"
#include "core/FS/file_system.h"
#include "core/FS/os_file.h"
#include "core/log.h"
#include "core/MT/task.h"
#include "core/path_utils.h"
#include "core/system.h"
#include "crnlib.h"
#include "debug/floating_points.h"
#include "editor/world_editor.h"
#include "engine/engine.h"
#include "metadata.h"
#include "imgui/imgui.h"
#include "platform_interface.h"
#include "physics/physics_geometry_manager.h"
#include "renderer/model.h"
#define STB_IMAGE_IMPLEMENTATION
#include "stb/stb_image.h"
#include "utils.h"


typedef StringBuilder<Lumix::MAX_PATH_LENGTH> PathBuilder;


enum class VertexAttributeDef : Lumix::uint32
{
	POSITION,
	FLOAT1,
	FLOAT2,
	FLOAT3,
	FLOAT4,
	INT1,
	INT2,
	INT3,
	INT4,
	SHORT2,
	SHORT4,
	BYTE4,
	NONE
};


struct DDSConvertCallbackData
{
	ImportAssetDialog* dialog;
	const char* dest_path;
};


crn_bool ddsConvertCallback(crn_uint32 phase_index,
	crn_uint32 total_phases,
	crn_uint32 subphase_index,
	crn_uint32 total_subphases,
	void* pUser_data_ptr)
{
	DDSConvertCallbackData* data = (DDSConvertCallbackData*)pUser_data_ptr;

	float fraction = phase_index / float(total_phases) +
					 (subphase_index / float(total_subphases)) / total_phases;
	data->dialog->setImportMessage(
		StringBuilder<Lumix::MAX_PATH_LENGTH + 50>("Saving ") << data->dest_path << "\n"
															  << int(fraction * 100)
															  << "%%");

	return true;
}


static bool saveAsRaw(ImportAssetDialog& dialog,
	Lumix::FS::FileSystem& fs,
	const Lumix::uint8* image_data,
	int image_width,
	int image_height,
	const char* dest_path,
	float scale,
	Lumix::IAllocator& allocator)
{
	ASSERT(image_data);

	dialog.setImportMessage(StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Saving ") << dest_path);

	auto* file = fs.open(fs.getDiskDevice(),
		Lumix::Path(dest_path),
		Lumix::FS::Mode::WRITE | Lumix::FS::Mode::CREATE);
	if (!file)
	{
		dialog.setMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not save ") << dest_path);
		return false;
	}

	Lumix::Array<Lumix::uint16> data(allocator);
	data.resize(image_width * image_height);
	for (int j = 0; j < image_height; ++j)
	{
		for (int i = 0; i < image_width; ++i)
		{
			data[i + j * image_width] =
				Lumix::uint16(scale * image_data[(i + j * image_width) * 4]);
		}
	}

	file->write((const char*)&data[0], data.size() * sizeof(data[0]));
	fs.close(*file);
	return true;
}


static bool saveAsDDS(ImportAssetDialog& dialog,
	Lumix::FS::FileSystem& fs,
	const char* source_path,
	const Lumix::uint8* image_data,
	int image_width,
	int image_height,
	const char* dest_path)
{
	ASSERT(image_data);

	dialog.setImportMessage(StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Saving ") << dest_path);

	DDSConvertCallbackData callback_data;
	callback_data.dialog = &dialog;
	callback_data.dest_path = dest_path;

	crn_uint32 size;
	crn_comp_params comp_params;
	comp_params.m_width = image_width;
	comp_params.m_height = image_height;
	comp_params.m_file_type = cCRNFileTypeDDS;
	comp_params.m_format = cCRNFmtDXT3;
	comp_params.m_quality_level = cCRNMinQualityLevel;
	comp_params.m_dxt_quality = cCRNDXTQualitySuperFast;
	comp_params.m_dxt_compressor_type = cCRNDXTCompressorRYG;
	comp_params.m_pProgress_func = ddsConvertCallback;
	comp_params.m_pProgress_func_data = &callback_data;
	comp_params.m_num_helper_threads = 3;
	comp_params.m_pImages[0][0] = (Lumix::uint32*)image_data;
	crn_mipmap_params mipmap_params;
	mipmap_params.m_mode = cCRNMipModeGenerateMips;

	void* data = crn_compress(comp_params, mipmap_params, size);
	if (!data)
	{
		dialog.setMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not convert ") << source_path);
		return false;
	}

	auto* file = fs.open(fs.getDiskDevice(),
		Lumix::Path(dest_path),
		Lumix::FS::Mode::WRITE | Lumix::FS::Mode::CREATE);
	if (!file)
	{
		dialog.setMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not save ") << dest_path);
		crn_free_block(data);
		return false;
	}

	file->write((const char*)data, size);
	fs.close(*file);
	crn_free_block(data);
	return true;
}


struct ImportTextureTask : public Lumix::MT::Task
{
	ImportTextureTask(ImportAssetDialog& dialog)
		: Task(dialog.m_editor.getAllocator())
		, m_dialog(dialog)
	{
	}


	static void getDestinationPath(const char* output_dir,
		const char* source,
		bool to_dds,
		bool to_raw,
		char* out,
		int max_size)
	{
		char basename[Lumix::MAX_PATH_LENGTH];
		Lumix::PathUtils::getBasename(basename, sizeof(basename), source);

		if (to_dds)
		{
			PathBuilder dest_path(output_dir);
			dest_path << "/" << basename << ".dds";
			Lumix::copyString(out, max_size, dest_path);
			return;
		}

		if (to_raw)
		{
			PathBuilder dest_path(output_dir);
			dest_path << "/" << basename << ".raw";
			Lumix::copyString(out, max_size, dest_path);
			return;
		}

		char ext[Lumix::MAX_PATH_LENGTH];
		Lumix::PathUtils::getExtension(ext, sizeof(ext), source);
		PathBuilder dest_path(output_dir);
		dest_path << "/" << basename << "." << ext;
		Lumix::copyString(out, max_size, dest_path);
	}


	int task() override
	{
		m_dialog.setImportMessage("Importing texture...");
		int image_width;
		int image_height;
		int image_comp;
		auto* data = stbi_load(m_dialog.m_source, &image_width, &image_height, &image_comp, 4);

		if (!data)
		{
			m_dialog.setMessage(
				StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not load ") << m_dialog.m_source);
			return -1;
		}

		char dest_path[Lumix::MAX_PATH_LENGTH];
		getDestinationPath(m_dialog.m_output_dir,
			m_dialog.m_source,
			m_dialog.m_convert_to_dds,
			m_dialog.m_convert_to_raw,
			dest_path,
			Lumix::lengthOf(dest_path));

		if (m_dialog.m_convert_to_dds)
		{
			m_dialog.setImportMessage("Converting to DDS...");

			saveAsDDS(m_dialog,
				m_dialog.m_editor.getEngine().getFileSystem(),
				m_dialog.m_source,
				data,
				image_width,
				image_height,
				dest_path);
		}
		else if (m_dialog.m_convert_to_raw)
		{
			m_dialog.setImportMessage("Converting to RAW...");

			saveAsRaw(m_dialog,
				m_dialog.m_editor.getEngine().getFileSystem(),
				data,
				image_width,
				image_height,
				dest_path,
				m_dialog.m_raw_texture_scale,
				m_dialog.m_editor.getAllocator());
		}
		else
		{
			m_dialog.setImportMessage("Copying...");

			if (!Lumix::copyFile(m_dialog.m_source, dest_path))
			{
				m_dialog.setMessage(
					StringBuilder<Lumix::MAX_PATH_LENGTH * 2 + 30>("Could not copy ")
					<< m_dialog.m_source
					<< " to "
					<< dest_path);
			}
		}
		stbi_image_free(data);

		return 0;
	}


	ImportAssetDialog& m_dialog;

}; // struct ImportTextureTask


struct ImportTask : public Lumix::MT::Task
{
	struct ProgressHandler : public Assimp::ProgressHandler
	{
		bool Update(float percentage) override
		{
			m_task->m_dialog.setImportMessage(
				StringBuilder<50>("Importing... ") << int(percentage * 100) << "%%");

			return true;
		}

		ImportTask* m_task;
	};

	ImportTask(ImportAssetDialog& dialog)
		: Task(dialog.m_editor.getAllocator())
		, m_dialog(dialog)
	{
		m_dialog.m_importer.SetProgressHandler(&m_progress_handler);
		struct MyStream : public Assimp::LogStream
		{
			void write(const char* message) { Lumix::g_log_warning.log("import") << message; }
		};
		const unsigned int severity = Assimp::Logger::Err;
		Assimp::DefaultLogger::create(
			ASSIMP_DEFAULT_LOG_NAME, Assimp::Logger::NORMAL, 0, nullptr);

		Assimp::DefaultLogger::get()->attachStream(new MyStream(), severity);
	}


	~ImportTask() { m_dialog.m_importer.SetProgressHandler(nullptr); }


	bool hasAnyMeshTangents()
	{
		auto* scene = m_dialog.m_importer.GetScene();
		for (unsigned int i = 0; i < scene->mNumMeshes; ++i)
		{
			if (scene->mMeshes[i]->mTangents) return true;
		}
		return false;
	}


	int task() override
	{
		m_progress_handler.m_task = this;
		Lumix::enableFloatingPointTraps(false);
		m_dialog.m_importer.SetPropertyInteger(
			AI_CONFIG_PP_RVC_FLAGS, aiComponent_LIGHTS | aiComponent_CAMERAS);
		unsigned int flags = aiProcess_JoinIdenticalVertices | aiProcess_RemoveComponent |
							 aiProcess_GenUVCoords | aiProcess_RemoveRedundantMaterials |
							 aiProcess_Triangulate | aiProcess_LimitBoneWeights |
							 aiProcess_OptimizeGraph | aiProcess_CalcTangentSpace;
		flags |= m_dialog.m_gen_smooth_normal ? aiProcess_GenSmoothNormals : aiProcess_GenNormals;
		flags |= m_dialog.m_optimize_mesh_on_import ? aiProcess_OptimizeMeshes : 0;
		const aiScene* scene = m_dialog.m_importer.ReadFile(m_dialog.m_source, flags);
		if (!scene || !scene->mMeshes || !hasAnyMeshTangents())
		{
			m_dialog.m_importer.FreeScene();
			m_dialog.setMessage(m_dialog.m_importer.GetErrorString());
			Lumix::g_log_error.log("import") << m_dialog.m_importer.GetErrorString();
		}
		else
		{
			m_dialog.m_mesh_mask.resize(scene->mNumMeshes);
			for (int i = 0; i < m_dialog.m_mesh_mask.size(); ++i)
			{
				m_dialog.m_mesh_mask[i] = scene->mMeshes[i]->mTangents != nullptr;
			}
		}

		Lumix::enableFloatingPointTraps(true);

		return 0;
	}


	ImportAssetDialog& m_dialog;
	ProgressHandler m_progress_handler;

}; // struct ImportTask


struct ConvertTask : public Lumix::MT::Task
{
	struct SkinInfo
	{
		SkinInfo()
		{
			memset(this, 0, sizeof(SkinInfo));
			index = 0;
		}

		float weights[4];
		Lumix::uint16 bone_indices[4];
		int index;
	};


	ConvertTask(ImportAssetDialog& dialog, float scale)
		: Task(dialog.m_editor.getAllocator())
		, m_dialog(dialog)
		, m_filtered_meshes(dialog.m_editor.getAllocator())
		, m_scale(scale)
	{
	}


	bool saveEmbeddedTextures(const aiScene* scene, unsigned int* materials, int materials_count)
	{
		bool success = true;
		m_dialog.m_saved_embedded_textures.clear();

		Lumix::Array<unsigned int> textures(m_dialog.m_editor.getAllocator());
		for (int i = 0; i < materials_count; ++i)
		{
			auto* material = scene->mMaterials[materials[i]];
			auto types = { aiTextureType_DIFFUSE, aiTextureType_NORMALS, aiTextureType_HEIGHT };
			for (auto type : types)
			{
				for (unsigned int j = 0; j < material->GetTextureCount(type); ++j)
				{
					aiString texture_path;
					material->GetTexture(type, j, &texture_path);
					if (texture_path.C_Str()[0] == '*')
					{
						unsigned int index;
						Lumix::fromCString(
							texture_path.C_Str() + 1, (int)texture_path.length - 1, &index);
						textures.push(index);
					}
				}
			}
		}

		for (unsigned int i = 0; i < scene->mNumTextures; ++i)
		{
			m_dialog.m_saved_embedded_textures.push(Lumix::string("", m_dialog.m_editor.getAllocator()));
			if (textures.indexOf(i) == -1) continue;
			
			const aiTexture* texture = scene->mTextures[i];
			if (texture->mHeight != 0)
			{
				m_dialog.setMessage("Uncompressed texture embedded. This is not supported.");
				return false;
			}
			PathBuilder texture_name("texture");
			texture_name << i << ".dds";
			int width, height, comp;
			auto data = stbi_load_from_memory(
				(stbi_uc*)texture->pcData, texture->mWidth, &width, &height, &comp, 4);
			if (!data) continue;

			m_dialog.m_saved_embedded_textures[i] = texture_name;
			PathBuilder dest(m_dialog.m_texture_output_dir[0] ? m_dialog.m_texture_output_dir
															  : m_dialog.m_output_dir);
			dest << "/" << texture_name;
			bool saved = saveAsDDS(m_dialog,
				m_dialog.m_editor.getEngine().getFileSystem(),
				"Embedded texture",
				data,
				width,
				height,
				dest);
			success = success && saved;

			stbi_image_free(data);
		}
		return success;
	}


	bool saveTexture(const char* texture_path,
		const char* source_mesh_dir,
		Lumix::FS::IFile& material_file) const
	{
		Lumix::string texture_source_path(texture_path, m_dialog.m_editor.getAllocator());
		int mapping_index = m_dialog.m_path_mapping.find(texture_source_path);
		if (mapping_index >= 0)
		{
			texture_source_path = m_dialog.m_path_mapping.at(mapping_index);
		}
		bool is_embedded = false;
		if (texture_source_path[0] == '*')
		{
			is_embedded = true;
			int index;
			Lumix::fromCString(
				texture_source_path.c_str() + 1, texture_source_path.length() - 1, &index);
			texture_source_path = m_dialog.m_saved_embedded_textures[index];
		}

		Lumix::PathUtils::FileInfo texture_info(texture_source_path.c_str());
		if (!m_dialog.m_texture_output_dir[0])
		{
			material_file << "\t, \"texture\" : {\n\t\t\"source\" : \"" << texture_info.m_basename
						  << ".";
			material_file << (m_dialog.m_convert_to_dds ? "dds" : texture_info.m_extension);
			material_file << "\"\n }\n";
		}
		else
		{
			material_file << "\t, \"texture\" : {\n\t\t\"source\" : \"";
			char from_root_path[Lumix::MAX_PATH_LENGTH];
			m_dialog.m_editor.getRelativePath(
				from_root_path, Lumix::lengthOf(from_root_path), m_dialog.m_texture_output_dir);
			material_file << "/" << from_root_path;
			material_file << texture_info.m_basename << ".";
			material_file << (m_dialog.m_convert_to_dds ? "dds" : texture_info.m_extension);
			material_file << "\"\n }\n";
		}

		bool is_already_saved = m_dialog.m_saved_textures.indexOf(texture_source_path) >= 0;
		if (is_embedded || is_already_saved) return true;

		PathBuilder source_absolute(source_mesh_dir);
		source_absolute << "/" << texture_source_path.c_str();
		const char* source = Lumix::PathUtils::isAbsolute(texture_source_path.c_str())
								 ? texture_source_path.c_str()
								 : source_absolute;

		if (m_dialog.m_convert_to_dds && Lumix::compareString(texture_info.m_extension, "dds") != 0)
		{
			PathBuilder dest(m_dialog.m_texture_output_dir[0] ? m_dialog.m_texture_output_dir
															  : m_dialog.m_output_dir);
			dest << "/" << texture_info.m_basename << ".dds";
			int image_width, image_height, dummy;
			auto data = stbi_load(source, &image_width, &image_height, &dummy, 4);
			if (!data)
			{
				StringBuilder<Lumix::MAX_PATH_LENGTH + 20> error_msg(
					"Could not load image ", source);
				m_dialog.setMessage(error_msg);
				return false;
			}

			if (!saveAsDDS(m_dialog,
					m_dialog.m_editor.getEngine().getFileSystem(),
					source,
					data,
					image_width,
					image_height,
					dest))
			{
				stbi_image_free(data);
				m_dialog.setMessage(StringBuilder<Lumix::MAX_PATH_LENGTH * 2 + 20>(
					"Error converting ", source, " to ", dest));
				return false;
			}
			stbi_image_free(data);
		}
		else
		{
			PathBuilder dest(m_dialog.m_output_dir);
			dest << "/" << texture_info.m_basename << "." << texture_info.m_extension;
			if (Lumix::compareString(source, dest) != 0 && !Lumix::copyFile(source, dest))
			{
				m_dialog.setMessage(StringBuilder<Lumix::MAX_PATH_LENGTH * 2 + 20>(
					"Error copying ", source, " to ", dest));
				return false;
			}
		}

		m_dialog.m_saved_textures.push(texture_source_path);
		return true;
	}


	bool saveLumixMaterials()
	{
		if (!m_dialog.m_import_materials) return true;

		m_dialog.setImportMessage("Importing materials...");
		const aiScene* scene = m_dialog.m_importer.GetScene();

		Lumix::Array<unsigned int> materials(m_dialog.m_editor.getAllocator());
		for (unsigned int i = 0; i < scene->mNumMeshes; ++i)
		{
			if (m_dialog.m_mesh_mask[i]) materials.push(scene->mMeshes[i]->mMaterialIndex);
		}
		materials.removeDuplicates();

		if (!saveEmbeddedTextures(scene, &materials[0], materials.size()))
		{
			m_dialog.setMessage("Failed to import embedded texture");
		}

		m_dialog.m_saved_textures.clear();

		int undefined_count = 0;
		char source_mesh_dir[Lumix::MAX_PATH_LENGTH];
		Lumix::PathUtils::getDir(source_mesh_dir, sizeof(source_mesh_dir), m_dialog.m_source);

		for (auto i : materials)
		{
			const aiMaterial* material = scene->mMaterials[i];
			if (!saveMaterial(material, source_mesh_dir, &undefined_count))
			{
				return false;
			}
		}
		return true;
	}


	bool saveMaterial(const aiMaterial* material,
		const char* source_mesh_dir,
		int* undefined_count) const
	{
		ASSERT(undefined_count);

		aiString material_name;
		material->Get(AI_MATKEY_NAME, material_name);
		PathBuilder output_material_name(m_dialog.m_output_dir);
		output_material_name << "/" << material_name.C_Str() << ".mat";

		m_dialog.setImportMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Converting ") << output_material_name);
		auto& fs = m_dialog.m_editor.getEngine().getFileSystem();
		auto* file = fs.open(fs.getDiskDevice(),
			Lumix::Path(output_material_name),
			Lumix::FS::Mode::CREATE | Lumix::FS::Mode::WRITE);
		if (!file)
		{
			m_dialog.setMessage(StringBuilder<20 + Lumix::MAX_PATH_LENGTH>(
				"Could not create ", output_material_name));
			return false;
		}

		const aiScene* scene = m_dialog.m_importer.GetScene();
		*file << "{\n\t\"shader\" : \"shaders/"
			  << (isSkinned(scene, material) ? "skinned" : "rigid") << ".shd\"\n";

		if (material->GetTextureCount(aiTextureType_DIFFUSE) == 1)
		{
			aiString texture_path;
			material->GetTexture(aiTextureType_DIFFUSE, 0, &texture_path);
			saveTexture(texture_path.C_Str(), source_mesh_dir, *file);
		}
		else
		{
			saveTexture(
				PathBuilder("undefined") << *undefined_count << ".dds", source_mesh_dir, *file);
			++*undefined_count;
		}

		if (material->GetTextureCount(aiTextureType_NORMALS) == 1)
		{
			aiString texture_path;
			material->GetTexture(aiTextureType_NORMALS, 0, &texture_path);
			saveTexture(texture_path.C_Str(), source_mesh_dir, *file);
		}
		else if (material->GetTextureCount(aiTextureType_HEIGHT) == 1)
		{
			aiString texture_path;
			material->GetTexture(aiTextureType_HEIGHT, 0, &texture_path);
			saveTexture(texture_path.C_Str(), source_mesh_dir, *file);
		}
		else if (material->GetTextureCount(aiTextureType_NORMALS) > 1)
		{
			m_dialog.setMessage(StringBuilder<Lumix::MAX_PATH_LENGTH + 20>(
				"Too many normal maps in ", material_name.C_Str()));
			fs.close(*file);
			return false;
		}
		file->write("}", 1);
		fs.close(*file);
		return true;
	}


	int task() override
	{
		if (saveLumixPhysics() && saveLumixModel() && saveLumixMaterials())
		{
			m_dialog.setMessage("Success.");
		}
		return 0;
	}


	static int countNodes(const aiNode* node)
	{
		int count = 1;
		for (unsigned int i = 0; i < node->mNumChildren; ++i)
		{
			count += countNodes(node->mChildren[i]);
		}
		return count;
	}


	static bool isSkinned(const aiMesh* mesh) { return mesh->mNumBones > 0; }


	static bool isSkinned(const aiScene* scene, const aiMaterial* material)
	{
		for (unsigned int i = 0; i < scene->mNumMeshes; ++i)
		{
			if (scene->mMaterials[scene->mMeshes[i]->mMaterialIndex] == material &&
				isSkinned(scene->mMeshes[i]))
			{
				return true;
			}
		}
		return false;
	}


	static void getBoneNamesHashes(const aiNode* node, Lumix::Array<int>& node_names)
	{
		node_names.push(Lumix::crc32(node->mName.C_Str()));
		for (unsigned int i = 0; i < node->mNumChildren; ++i)
		{
			getBoneNamesHashes(node->mChildren[i], node_names);
		}
	}


	void fillSkinInfo(const aiScene* scene, Lumix::Array<SkinInfo>& infos, int vertices_count) const
	{

		Lumix::Array<int> node_names(m_dialog.m_editor.getAllocator());
		getBoneNamesHashes(scene->mRootNode, node_names);
		infos.resize(vertices_count);

		int offset = 0;
		for (auto* mesh : m_filtered_meshes)
		{
			for (unsigned int j = 0; j < mesh->mNumBones; ++j)
			{
				const aiBone* bone = mesh->mBones[j];
				int bone_index = node_names.indexOf(Lumix::crc32(bone->mName.C_Str()));
				for (unsigned int k = 0; k < bone->mNumWeights; ++k)
				{
					auto& info = infos[offset + bone->mWeights[k].mVertexId];
					info.weights[info.index] = bone->mWeights[k].mWeight;
					info.bone_indices[info.index] = bone_index;
					++info.index;
				}
			}
			offset += mesh->mNumVertices;
		}
	}


	static Lumix::uint32 packuint32(Lumix::uint8 _x,
		Lumix::uint8 _y,
		Lumix::uint8 _z,
		Lumix::uint8 _w)
	{
		union
		{
			Lumix::uint32 ui32;
			Lumix::uint8 arr[4];
		} un;

		un.arr[0] = _x;
		un.arr[1] = _y;
		un.arr[2] = _z;
		un.arr[3] = _w;

		return un.ui32;
	}


	static Lumix::uint32 packF4u(const aiVector3D& vec)
	{
		const Lumix::uint8 xx = Lumix::uint8(vec.x * 127.0f + 128.0f);
		const Lumix::uint8 yy = Lumix::uint8(vec.y * 127.0f + 128.0f);
		const Lumix::uint8 zz = Lumix::uint8(vec.z * 127.0f + 128.0f);
		const Lumix::uint8 ww = Lumix::uint8(0);
		return packuint32(xx, yy, zz, ww);
	}


	void writeGeometry(Lumix::FS::IFile& file) const
	{
		const aiScene* scene = m_dialog.m_importer.GetScene();
		Lumix::int32 indices_count = 0;
		int vertices_count = 0;
		Lumix::int32 vertices_size = 0;
		for (auto* mesh : m_filtered_meshes)
		{
			indices_count += mesh->mNumFaces * 3;
			vertices_count += mesh->mNumVertices;
			vertices_size += mesh->mNumVertices * getVertexSize(mesh);
		}

		file.write((const char*)&indices_count, sizeof(indices_count));
		Lumix::int32 polygon_idx = 0;
		for (auto* mesh : m_filtered_meshes)
		{
			for (unsigned int j = 0; j < mesh->mNumFaces; ++j)
			{
				polygon_idx = mesh->mFaces[j].mIndices[0];
				file.write((const char*)&polygon_idx, sizeof(polygon_idx));
				polygon_idx = mesh->mFaces[j].mIndices[1];
				file.write((const char*)&polygon_idx, sizeof(polygon_idx));
				polygon_idx = mesh->mFaces[j].mIndices[2];
				file.write((const char*)&polygon_idx, sizeof(polygon_idx));
			}
		}

		file.write((const char*)&vertices_size, sizeof(vertices_size));

		Lumix::Array<SkinInfo> skin_infos(m_dialog.m_editor.getAllocator());
		fillSkinInfo(scene, skin_infos, vertices_count);

		int skin_index = 0;
		aiMatrix3x3 normal_matrix(scene->mRootNode->mTransformation);

		bool z_up = m_dialog.m_z_up;
		for (auto* mesh : m_filtered_meshes)
		{
			bool is_skinned = isSkinned(mesh);
			for (unsigned int j = 0; j < mesh->mNumVertices; ++j)
			{
				if (is_skinned)
				{
					file.write((const char*)skin_infos[skin_index].weights,
						sizeof(skin_infos[skin_index].weights));
					file.write((const char*)skin_infos[skin_index].bone_indices,
						sizeof(skin_infos[skin_index].bone_indices));
				}
				++skin_index;

				auto v = scene->mRootNode->mTransformation * mesh->mVertices[j];

				Lumix::Vec3 position(v.x, z_up ? v.z : v.y, z_up ? -v.y : v.z);
				position *= m_scale;
				file.write((const char*)&position, sizeof(position));

				if (mesh->mColors[0])
				{
					auto assimp_color = mesh->mColors[0][j];
					Lumix::uint8 color[4];
					color[0] = Lumix::uint8(assimp_color.r * 255);
					color[1] = Lumix::uint8(assimp_color.g * 255);
					color[2] = Lumix::uint8(assimp_color.b * 255);
					color[3] = Lumix::uint8(assimp_color.a * 255);
					file.write(color, sizeof(color));
				}

				auto normal = normal_matrix * mesh->mNormals[j];
				normal.Normalize();
				if (z_up) normal.Set(normal.x, normal.z, -normal.y);
				Lumix::uint32 int_normal = packF4u(normal);
				file.write((const char*)&int_normal, sizeof(int_normal));

				if (mesh->mTangents)
				{
					auto tangent = mesh->mTangents[j];
					if (z_up) tangent.Set(tangent.x, tangent.z, -tangent.y);
					Lumix::uint32 int_tangent = packF4u(tangent);
					file.write((const char*)&int_tangent, sizeof(int_tangent));
				}
<commit_msg>fixed importing tangents<commit_after>#include "import_asset_dialog.h"
#include "assimp/DefaultLogger.hpp"
#include "assimp/postprocess.h"
#include "assimp/ProgressHandler.hpp"
#include "assimp/scene.h"
#include "core/crc32.h"
#include "core/FS/ifile.h"
#include "core/FS/file_system.h"
#include "core/FS/os_file.h"
#include "core/log.h"
#include "core/MT/task.h"
#include "core/path_utils.h"
#include "core/system.h"
#include "crnlib.h"
#include "debug/floating_points.h"
#include "editor/world_editor.h"
#include "engine/engine.h"
#include "metadata.h"
#include "imgui/imgui.h"
#include "platform_interface.h"
#include "physics/physics_geometry_manager.h"
#include "renderer/model.h"
#define STB_IMAGE_IMPLEMENTATION
#include "stb/stb_image.h"
#include "utils.h"


typedef StringBuilder<Lumix::MAX_PATH_LENGTH> PathBuilder;


enum class VertexAttributeDef : Lumix::uint32
{
	POSITION,
	FLOAT1,
	FLOAT2,
	FLOAT3,
	FLOAT4,
	INT1,
	INT2,
	INT3,
	INT4,
	SHORT2,
	SHORT4,
	BYTE4,
	NONE
};


struct DDSConvertCallbackData
{
	ImportAssetDialog* dialog;
	const char* dest_path;
};


crn_bool ddsConvertCallback(crn_uint32 phase_index,
	crn_uint32 total_phases,
	crn_uint32 subphase_index,
	crn_uint32 total_subphases,
	void* pUser_data_ptr)
{
	DDSConvertCallbackData* data = (DDSConvertCallbackData*)pUser_data_ptr;

	float fraction = phase_index / float(total_phases) +
					 (subphase_index / float(total_subphases)) / total_phases;
	data->dialog->setImportMessage(
		StringBuilder<Lumix::MAX_PATH_LENGTH + 50>("Saving ") << data->dest_path << "\n"
															  << int(fraction * 100)
															  << "%%");

	return true;
}


static bool saveAsRaw(ImportAssetDialog& dialog,
	Lumix::FS::FileSystem& fs,
	const Lumix::uint8* image_data,
	int image_width,
	int image_height,
	const char* dest_path,
	float scale,
	Lumix::IAllocator& allocator)
{
	ASSERT(image_data);

	dialog.setImportMessage(StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Saving ") << dest_path);

	auto* file = fs.open(fs.getDiskDevice(),
		Lumix::Path(dest_path),
		Lumix::FS::Mode::WRITE | Lumix::FS::Mode::CREATE);
	if (!file)
	{
		dialog.setMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not save ") << dest_path);
		return false;
	}

	Lumix::Array<Lumix::uint16> data(allocator);
	data.resize(image_width * image_height);
	for (int j = 0; j < image_height; ++j)
	{
		for (int i = 0; i < image_width; ++i)
		{
			data[i + j * image_width] =
				Lumix::uint16(scale * image_data[(i + j * image_width) * 4]);
		}
	}

	file->write((const char*)&data[0], data.size() * sizeof(data[0]));
	fs.close(*file);
	return true;
}


static bool saveAsDDS(ImportAssetDialog& dialog,
	Lumix::FS::FileSystem& fs,
	const char* source_path,
	const Lumix::uint8* image_data,
	int image_width,
	int image_height,
	const char* dest_path)
{
	ASSERT(image_data);

	dialog.setImportMessage(StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Saving ") << dest_path);

	DDSConvertCallbackData callback_data;
	callback_data.dialog = &dialog;
	callback_data.dest_path = dest_path;

	crn_uint32 size;
	crn_comp_params comp_params;
	comp_params.m_width = image_width;
	comp_params.m_height = image_height;
	comp_params.m_file_type = cCRNFileTypeDDS;
	comp_params.m_format = cCRNFmtDXT3;
	comp_params.m_quality_level = cCRNMinQualityLevel;
	comp_params.m_dxt_quality = cCRNDXTQualitySuperFast;
	comp_params.m_dxt_compressor_type = cCRNDXTCompressorRYG;
	comp_params.m_pProgress_func = ddsConvertCallback;
	comp_params.m_pProgress_func_data = &callback_data;
	comp_params.m_num_helper_threads = 3;
	comp_params.m_pImages[0][0] = (Lumix::uint32*)image_data;
	crn_mipmap_params mipmap_params;
	mipmap_params.m_mode = cCRNMipModeGenerateMips;

	void* data = crn_compress(comp_params, mipmap_params, size);
	if (!data)
	{
		dialog.setMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not convert ") << source_path);
		return false;
	}

	auto* file = fs.open(fs.getDiskDevice(),
		Lumix::Path(dest_path),
		Lumix::FS::Mode::WRITE | Lumix::FS::Mode::CREATE);
	if (!file)
	{
		dialog.setMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not save ") << dest_path);
		crn_free_block(data);
		return false;
	}

	file->write((const char*)data, size);
	fs.close(*file);
	crn_free_block(data);
	return true;
}


struct ImportTextureTask : public Lumix::MT::Task
{
	ImportTextureTask(ImportAssetDialog& dialog)
		: Task(dialog.m_editor.getAllocator())
		, m_dialog(dialog)
	{
	}


	static void getDestinationPath(const char* output_dir,
		const char* source,
		bool to_dds,
		bool to_raw,
		char* out,
		int max_size)
	{
		char basename[Lumix::MAX_PATH_LENGTH];
		Lumix::PathUtils::getBasename(basename, sizeof(basename), source);

		if (to_dds)
		{
			PathBuilder dest_path(output_dir);
			dest_path << "/" << basename << ".dds";
			Lumix::copyString(out, max_size, dest_path);
			return;
		}

		if (to_raw)
		{
			PathBuilder dest_path(output_dir);
			dest_path << "/" << basename << ".raw";
			Lumix::copyString(out, max_size, dest_path);
			return;
		}

		char ext[Lumix::MAX_PATH_LENGTH];
		Lumix::PathUtils::getExtension(ext, sizeof(ext), source);
		PathBuilder dest_path(output_dir);
		dest_path << "/" << basename << "." << ext;
		Lumix::copyString(out, max_size, dest_path);
	}


	int task() override
	{
		m_dialog.setImportMessage("Importing texture...");
		int image_width;
		int image_height;
		int image_comp;
		auto* data = stbi_load(m_dialog.m_source, &image_width, &image_height, &image_comp, 4);

		if (!data)
		{
			m_dialog.setMessage(
				StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Could not load ") << m_dialog.m_source);
			return -1;
		}

		char dest_path[Lumix::MAX_PATH_LENGTH];
		getDestinationPath(m_dialog.m_output_dir,
			m_dialog.m_source,
			m_dialog.m_convert_to_dds,
			m_dialog.m_convert_to_raw,
			dest_path,
			Lumix::lengthOf(dest_path));

		if (m_dialog.m_convert_to_dds)
		{
			m_dialog.setImportMessage("Converting to DDS...");

			saveAsDDS(m_dialog,
				m_dialog.m_editor.getEngine().getFileSystem(),
				m_dialog.m_source,
				data,
				image_width,
				image_height,
				dest_path);
		}
		else if (m_dialog.m_convert_to_raw)
		{
			m_dialog.setImportMessage("Converting to RAW...");

			saveAsRaw(m_dialog,
				m_dialog.m_editor.getEngine().getFileSystem(),
				data,
				image_width,
				image_height,
				dest_path,
				m_dialog.m_raw_texture_scale,
				m_dialog.m_editor.getAllocator());
		}
		else
		{
			m_dialog.setImportMessage("Copying...");

			if (!Lumix::copyFile(m_dialog.m_source, dest_path))
			{
				m_dialog.setMessage(
					StringBuilder<Lumix::MAX_PATH_LENGTH * 2 + 30>("Could not copy ")
					<< m_dialog.m_source
					<< " to "
					<< dest_path);
			}
		}
		stbi_image_free(data);

		return 0;
	}


	ImportAssetDialog& m_dialog;

}; // struct ImportTextureTask


struct ImportTask : public Lumix::MT::Task
{
	struct ProgressHandler : public Assimp::ProgressHandler
	{
		bool Update(float percentage) override
		{
			m_task->m_dialog.setImportMessage(
				StringBuilder<50>("Importing... ") << int(percentage * 100) << "%%");

			return true;
		}

		ImportTask* m_task;
	};

	ImportTask(ImportAssetDialog& dialog)
		: Task(dialog.m_editor.getAllocator())
		, m_dialog(dialog)
	{
		m_dialog.m_importer.SetProgressHandler(&m_progress_handler);
		struct MyStream : public Assimp::LogStream
		{
			void write(const char* message) { Lumix::g_log_warning.log("import") << message; }
		};
		const unsigned int severity = Assimp::Logger::Err;
		Assimp::DefaultLogger::create(
			ASSIMP_DEFAULT_LOG_NAME, Assimp::Logger::NORMAL, 0, nullptr);

		Assimp::DefaultLogger::get()->attachStream(new MyStream(), severity);
	}


	~ImportTask() { m_dialog.m_importer.SetProgressHandler(nullptr); }


	bool hasAnyMeshTangents()
	{
		auto* scene = m_dialog.m_importer.GetScene();
		for (unsigned int i = 0; i < scene->mNumMeshes; ++i)
		{
			if (scene->mMeshes[i]->mTangents) return true;
		}
		return false;
	}


	int task() override
	{
		m_progress_handler.m_task = this;
		Lumix::enableFloatingPointTraps(false);
		m_dialog.m_importer.SetPropertyInteger(
			AI_CONFIG_PP_RVC_FLAGS, aiComponent_LIGHTS | aiComponent_CAMERAS);
		unsigned int flags = aiProcess_JoinIdenticalVertices | aiProcess_RemoveComponent |
							 aiProcess_GenUVCoords | aiProcess_RemoveRedundantMaterials |
							 aiProcess_Triangulate | aiProcess_LimitBoneWeights |
							 aiProcess_OptimizeGraph | aiProcess_CalcTangentSpace;
		flags |= m_dialog.m_gen_smooth_normal ? aiProcess_GenSmoothNormals : aiProcess_GenNormals;
		flags |= m_dialog.m_optimize_mesh_on_import ? aiProcess_OptimizeMeshes : 0;
		const aiScene* scene = m_dialog.m_importer.ReadFile(m_dialog.m_source, flags);
		if (!scene || !scene->mMeshes || !hasAnyMeshTangents())
		{
			m_dialog.m_importer.FreeScene();
			m_dialog.setMessage(m_dialog.m_importer.GetErrorString());
			Lumix::g_log_error.log("import") << m_dialog.m_importer.GetErrorString();
		}
		else
		{
			m_dialog.m_mesh_mask.resize(scene->mNumMeshes);
			for (int i = 0; i < m_dialog.m_mesh_mask.size(); ++i)
			{
				m_dialog.m_mesh_mask[i] = scene->mMeshes[i]->mTangents != nullptr;
			}
		}

		Lumix::enableFloatingPointTraps(true);

		return 0;
	}


	ImportAssetDialog& m_dialog;
	ProgressHandler m_progress_handler;

}; // struct ImportTask


struct ConvertTask : public Lumix::MT::Task
{
	struct SkinInfo
	{
		SkinInfo()
		{
			memset(this, 0, sizeof(SkinInfo));
			index = 0;
		}

		float weights[4];
		Lumix::uint16 bone_indices[4];
		int index;
	};


	ConvertTask(ImportAssetDialog& dialog, float scale)
		: Task(dialog.m_editor.getAllocator())
		, m_dialog(dialog)
		, m_filtered_meshes(dialog.m_editor.getAllocator())
		, m_scale(scale)
	{
	}


	bool saveEmbeddedTextures(const aiScene* scene, unsigned int* materials, int materials_count)
	{
		bool success = true;
		m_dialog.m_saved_embedded_textures.clear();

		Lumix::Array<unsigned int> textures(m_dialog.m_editor.getAllocator());
		for (int i = 0; i < materials_count; ++i)
		{
			auto* material = scene->mMaterials[materials[i]];
			auto types = { aiTextureType_DIFFUSE, aiTextureType_NORMALS, aiTextureType_HEIGHT };
			for (auto type : types)
			{
				for (unsigned int j = 0; j < material->GetTextureCount(type); ++j)
				{
					aiString texture_path;
					material->GetTexture(type, j, &texture_path);
					if (texture_path.C_Str()[0] == '*')
					{
						unsigned int index;
						Lumix::fromCString(
							texture_path.C_Str() + 1, (int)texture_path.length - 1, &index);
						textures.push(index);
					}
				}
			}
		}

		for (unsigned int i = 0; i < scene->mNumTextures; ++i)
		{
			m_dialog.m_saved_embedded_textures.push(Lumix::string("", m_dialog.m_editor.getAllocator()));
			if (textures.indexOf(i) == -1) continue;
			
			const aiTexture* texture = scene->mTextures[i];
			if (texture->mHeight != 0)
			{
				m_dialog.setMessage("Uncompressed texture embedded. This is not supported.");
				return false;
			}
			PathBuilder texture_name("texture");
			texture_name << i << ".dds";
			int width, height, comp;
			auto data = stbi_load_from_memory(
				(stbi_uc*)texture->pcData, texture->mWidth, &width, &height, &comp, 4);
			if (!data) continue;

			m_dialog.m_saved_embedded_textures[i] = texture_name;
			PathBuilder dest(m_dialog.m_texture_output_dir[0] ? m_dialog.m_texture_output_dir
															  : m_dialog.m_output_dir);
			dest << "/" << texture_name;
			bool saved = saveAsDDS(m_dialog,
				m_dialog.m_editor.getEngine().getFileSystem(),
				"Embedded texture",
				data,
				width,
				height,
				dest);
			success = success && saved;

			stbi_image_free(data);
		}
		return success;
	}


	bool saveTexture(const char* texture_path,
		const char* source_mesh_dir,
		Lumix::FS::IFile& material_file) const
	{
		Lumix::string texture_source_path(texture_path, m_dialog.m_editor.getAllocator());
		int mapping_index = m_dialog.m_path_mapping.find(texture_source_path);
		if (mapping_index >= 0)
		{
			texture_source_path = m_dialog.m_path_mapping.at(mapping_index);
		}
		bool is_embedded = false;
		if (texture_source_path[0] == '*')
		{
			is_embedded = true;
			int index;
			Lumix::fromCString(
				texture_source_path.c_str() + 1, texture_source_path.length() - 1, &index);
			texture_source_path = m_dialog.m_saved_embedded_textures[index];
		}

		Lumix::PathUtils::FileInfo texture_info(texture_source_path.c_str());
		if (!m_dialog.m_texture_output_dir[0])
		{
			material_file << "\t, \"texture\" : {\n\t\t\"source\" : \"" << texture_info.m_basename
						  << ".";
			material_file << (m_dialog.m_convert_to_dds ? "dds" : texture_info.m_extension);
			material_file << "\"\n }\n";
		}
		else
		{
			material_file << "\t, \"texture\" : {\n\t\t\"source\" : \"";
			char from_root_path[Lumix::MAX_PATH_LENGTH];
			m_dialog.m_editor.getRelativePath(
				from_root_path, Lumix::lengthOf(from_root_path), m_dialog.m_texture_output_dir);
			material_file << "/" << from_root_path;
			material_file << texture_info.m_basename << ".";
			material_file << (m_dialog.m_convert_to_dds ? "dds" : texture_info.m_extension);
			material_file << "\"\n }\n";
		}

		bool is_already_saved = m_dialog.m_saved_textures.indexOf(texture_source_path) >= 0;
		if (is_embedded || is_already_saved) return true;

		PathBuilder source_absolute(source_mesh_dir);
		source_absolute << "/" << texture_source_path.c_str();
		const char* source = Lumix::PathUtils::isAbsolute(texture_source_path.c_str())
								 ? texture_source_path.c_str()
								 : source_absolute;

		if (m_dialog.m_convert_to_dds && Lumix::compareString(texture_info.m_extension, "dds") != 0)
		{
			PathBuilder dest(m_dialog.m_texture_output_dir[0] ? m_dialog.m_texture_output_dir
															  : m_dialog.m_output_dir);
			dest << "/" << texture_info.m_basename << ".dds";
			int image_width, image_height, dummy;
			auto data = stbi_load(source, &image_width, &image_height, &dummy, 4);
			if (!data)
			{
				StringBuilder<Lumix::MAX_PATH_LENGTH + 20> error_msg(
					"Could not load image ", source);
				m_dialog.setMessage(error_msg);
				return false;
			}

			if (!saveAsDDS(m_dialog,
					m_dialog.m_editor.getEngine().getFileSystem(),
					source,
					data,
					image_width,
					image_height,
					dest))
			{
				stbi_image_free(data);
				m_dialog.setMessage(StringBuilder<Lumix::MAX_PATH_LENGTH * 2 + 20>(
					"Error converting ", source, " to ", dest));
				return false;
			}
			stbi_image_free(data);
		}
		else
		{
			PathBuilder dest(m_dialog.m_output_dir);
			dest << "/" << texture_info.m_basename << "." << texture_info.m_extension;
			if (Lumix::compareString(source, dest) != 0 && !Lumix::copyFile(source, dest))
			{
				m_dialog.setMessage(StringBuilder<Lumix::MAX_PATH_LENGTH * 2 + 20>(
					"Error copying ", source, " to ", dest));
				return false;
			}
		}

		m_dialog.m_saved_textures.push(texture_source_path);
		return true;
	}


	bool saveLumixMaterials()
	{
		if (!m_dialog.m_import_materials) return true;

		m_dialog.setImportMessage("Importing materials...");
		const aiScene* scene = m_dialog.m_importer.GetScene();

		Lumix::Array<unsigned int> materials(m_dialog.m_editor.getAllocator());
		for (unsigned int i = 0; i < scene->mNumMeshes; ++i)
		{
			if (m_dialog.m_mesh_mask[i]) materials.push(scene->mMeshes[i]->mMaterialIndex);
		}
		materials.removeDuplicates();

		if (!saveEmbeddedTextures(scene, &materials[0], materials.size()))
		{
			m_dialog.setMessage("Failed to import embedded texture");
		}

		m_dialog.m_saved_textures.clear();

		int undefined_count = 0;
		char source_mesh_dir[Lumix::MAX_PATH_LENGTH];
		Lumix::PathUtils::getDir(source_mesh_dir, sizeof(source_mesh_dir), m_dialog.m_source);

		for (auto i : materials)
		{
			const aiMaterial* material = scene->mMaterials[i];
			if (!saveMaterial(material, source_mesh_dir, &undefined_count))
			{
				return false;
			}
		}
		return true;
	}


	bool saveMaterial(const aiMaterial* material,
		const char* source_mesh_dir,
		int* undefined_count) const
	{
		ASSERT(undefined_count);

		aiString material_name;
		material->Get(AI_MATKEY_NAME, material_name);
		PathBuilder output_material_name(m_dialog.m_output_dir);
		output_material_name << "/" << material_name.C_Str() << ".mat";

		m_dialog.setImportMessage(
			StringBuilder<Lumix::MAX_PATH_LENGTH + 30>("Converting ") << output_material_name);
		auto& fs = m_dialog.m_editor.getEngine().getFileSystem();
		auto* file = fs.open(fs.getDiskDevice(),
			Lumix::Path(output_material_name),
			Lumix::FS::Mode::CREATE | Lumix::FS::Mode::WRITE);
		if (!file)
		{
			m_dialog.setMessage(StringBuilder<20 + Lumix::MAX_PATH_LENGTH>(
				"Could not create ", output_material_name));
			return false;
		}

		const aiScene* scene = m_dialog.m_importer.GetScene();
		*file << "{\n\t\"shader\" : \"shaders/"
			  << (isSkinned(scene, material) ? "skinned" : "rigid") << ".shd\"\n";

		if (material->GetTextureCount(aiTextureType_DIFFUSE) == 1)
		{
			aiString texture_path;
			material->GetTexture(aiTextureType_DIFFUSE, 0, &texture_path);
			saveTexture(texture_path.C_Str(), source_mesh_dir, *file);
		}
		else
		{
			saveTexture(
				PathBuilder("undefined") << *undefined_count << ".dds", source_mesh_dir, *file);
			++*undefined_count;
		}

		if (material->GetTextureCount(aiTextureType_NORMALS) == 1)
		{
			aiString texture_path;
			material->GetTexture(aiTextureType_NORMALS, 0, &texture_path);
			saveTexture(texture_path.C_Str(), source_mesh_dir, *file);
		}
		else if (material->GetTextureCount(aiTextureType_HEIGHT) == 1)
		{
			aiString texture_path;
			material->GetTexture(aiTextureType_HEIGHT, 0, &texture_path);
			saveTexture(texture_path.C_Str(), source_mesh_dir, *file);
		}
		else if (material->GetTextureCount(aiTextureType_NORMALS) > 1)
		{
			m_dialog.setMessage(StringBuilder<Lumix::MAX_PATH_LENGTH + 20>(
				"Too many normal maps in ", material_name.C_Str()));
			fs.close(*file);
			return false;
		}
		file->write("}", 1);
		fs.close(*file);
		return true;
	}


	int task() override
	{
		if (saveLumixPhysics() && saveLumixModel() && saveLumixMaterials())
		{
			m_dialog.setMessage("Success.");
		}
		return 0;
	}


	static int countNodes(const aiNode* node)
	{
		int count = 1;
		for (unsigned int i = 0; i < node->mNumChildren; ++i)
		{
			count += countNodes(node->mChildren[i]);
		}
		return count;
	}


	static bool isSkinned(const aiMesh* mesh) { return mesh->mNumBones > 0; }


	static bool isSkinned(const aiScene* scene, const aiMaterial* material)
	{
		for (unsigned int i = 0; i < scene->mNumMeshes; ++i)
		{
			if (scene->mMaterials[scene->mMeshes[i]->mMaterialIndex] == material &&
				isSkinned(scene->mMeshes[i]))
			{
				return true;
			}
		}
		return false;
	}


	static void getBoneNamesHashes(const aiNode* node, Lumix::Array<int>& node_names)
	{
		node_names.push(Lumix::crc32(node->mName.C_Str()));
		for (unsigned int i = 0; i < node->mNumChildren; ++i)
		{
			getBoneNamesHashes(node->mChildren[i], node_names);
		}
	}


	void fillSkinInfo(const aiScene* scene, Lumix::Array<SkinInfo>& infos, int vertices_count) const
	{

		Lumix::Array<int> node_names(m_dialog.m_editor.getAllocator());
		getBoneNamesHashes(scene->mRootNode, node_names);
		infos.resize(vertices_count);

		int offset = 0;
		for (auto* mesh : m_filtered_meshes)
		{
			for (unsigned int j = 0; j < mesh->mNumBones; ++j)
			{
				const aiBone* bone = mesh->mBones[j];
				int bone_index = node_names.indexOf(Lumix::crc32(bone->mName.C_Str()));
				for (unsigned int k = 0; k < bone->mNumWeights; ++k)
				{
					auto& info = infos[offset + bone->mWeights[k].mVertexId];
					info.weights[info.index] = bone->mWeights[k].mWeight;
					info.bone_indices[info.index] = bone_index;
					++info.index;
				}
			}
			offset += mesh->mNumVertices;
		}
	}


	static Lumix::uint32 packuint32(Lumix::uint8 _x,
		Lumix::uint8 _y,
		Lumix::uint8 _z,
		Lumix::uint8 _w)
	{
		union
		{
			Lumix::uint32 ui32;
			Lumix::uint8 arr[4];
		} un;

		un.arr[0] = _x;
		un.arr[1] = _y;
		un.arr[2] = _z;
		un.arr[3] = _w;

		return un.ui32;
	}


	static Lumix::uint32 packF4u(const aiVector3D& vec)
	{
		const Lumix::uint8 xx = Lumix::uint8(vec.x * 127.0f + 128.0f);
		const Lumix::uint8 yy = Lumix::uint8(vec.y * 127.0f + 128.0f);
		const Lumix::uint8 zz = Lumix::uint8(vec.z * 127.0f + 128.0f);
		const Lumix::uint8 ww = Lumix::uint8(0);
		return packuint32(xx, yy, zz, ww);
	}


	void writeGeometry(Lumix::FS::IFile& file) const
	{
		const aiScene* scene = m_dialog.m_importer.GetScene();
		Lumix::int32 indices_count = 0;
		int vertices_count = 0;
		Lumix::int32 vertices_size = 0;
		for (auto* mesh : m_filtered_meshes)
		{
			indices_count += mesh->mNumFaces * 3;
			vertices_count += mesh->mNumVertices;
			vertices_size += mesh->mNumVertices * getVertexSize(mesh);
		}

		file.write((const char*)&indices_count, sizeof(indices_count));
		Lumix::int32 polygon_idx = 0;
		for (auto* mesh : m_filtered_meshes)
		{
			for (unsigned int j = 0; j < mesh->mNumFaces; ++j)
			{
				polygon_idx = mesh->mFaces[j].mIndices[0];
				file.write((const char*)&polygon_idx, sizeof(polygon_idx));
				polygon_idx = mesh->mFaces[j].mIndices[1];
				file.write((const char*)&polygon_idx, sizeof(polygon_idx));
				polygon_idx = mesh->mFaces[j].mIndices[2];
				file.write((const char*)&polygon_idx, sizeof(polygon_idx));
			}
		}

		file.write((const char*)&vertices_size, sizeof(vertices_size));

		Lumix::Array<SkinInfo> skin_infos(m_dialog.m_editor.getAllocator());
		fillSkinInfo(scene, skin_infos, vertices_count);

		int skin_index = 0;
		aiMatrix3x3 normal_matrix(scene->mRootNode->mTransformation);

		bool z_up = m_dialog.m_z_up;
		for (auto* mesh : m_filtered_meshes)
		{
			bool is_skinned = isSkinned(mesh);
			for (unsigned int j = 0; j < mesh->mNumVertices; ++j)
			{
				if (is_skinned)
				{
					file.write((const char*)skin_infos[skin_index].weights,
						sizeof(skin_infos[skin_index].weights));
					file.write((const char*)skin_infos[skin_index].bone_indices,
						sizeof(skin_infos[skin_index].bone_indices));
				}
				++skin_index;

				auto v = scene->mRootNode->mTransformation * mesh->mVertices[j];

				Lumix::Vec3 position(v.x, z_up ? v.z : v.y, z_up ? -v.y : v.z);
				position *= m_scale;
				file.write((const char*)&position, sizeof(position));

				if (mesh->mColors[0])
				{
					auto assimp_color = mesh->mColors[0][j];
					Lumix::uint8 color[4];
					color[0] = Lumix::uint8(assimp_color.r * 255);
					color[1] = Lumix::uint8(assimp_color.g * 255);
					color[2] = Lumix::uint8(assimp_color.b * 255);
					color[3] = Lumix::uint8(assimp_color.a * 255);
					file.write(color, sizeof(color));
				}

				auto normal = normal_matrix * mesh->mNormals[j];
				normal.Normalize();
				if (z_up) normal.Set(normal.x, normal.z, -normal.y);
				Lumix::uint32 int_normal = packF4u(normal);
				file.write((const char*)&int_normal, sizeof(int_normal));

				if (mesh->mTangents)
				{
					auto tangent = normal_matrix * mesh->mTangents[j];
					tangent.Normalize();
					if (z_up) tangent.Set(tangent.x, tangent.z, -tangent.y);
					Lumix::uint32 int_tangent = packF4u(tangent);
					file.write((const char*)&int_tangent, sizeof(int_tangent));
				}

{'fraction_non_alphanumeric': 0.14203966619703046, 'fraction_numerical': 0.01075105668147827, 'mean_word_length': 4.3683965557365605, 'pattern_counts': {'":': 0, '<': 323, '<?xml version=': 0, '>': 201, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4313526', 'n_tokens_mistral': 19988, 'n_tokens_neox': 17230, 'n_words': 2878}