<commit_before>#ifndef SIGMA_TOOLS_MODEL_LOADER_HPP
#define SIGMA_TOOLS_MODEL_LOADER_HPP

#include <sigma/blueprint.hpp>
#include <sigma/graphics/static_mesh.hpp>
#include <sigma/graphics/static_mesh_instance.hpp>
#include <sigma/graphics/technique.hpp>
#include <sigma/tools/packager.hpp>
#include <sigma/transform.hpp>
#include <sigma/util/filesystem.hpp>
#include <sigma/util/json_conversion.hpp>
#include <sigma/util/type_sequence.hpp>

#include <assimp/Importer.hpp>
#include <assimp/postprocess.h>
#include <assimp/scene.h>

#include <boost/algorithm/string.hpp>

#include <iostream>
#include <set>
#include <string>

namespace sigma {
namespace tools {
    glm::vec3 convert_color(aiColor3D c)
    {
        return glm::vec3(c.r, c.g, c.b);
    }

    glm::vec3 convert_3d(aiVector3D v)
    {
        return { v.x, v.y, v.z };
    }

    glm::quat convert(aiQuaternion q)
    {
        return { q.w, q.x, q.y, q.z };
    }

    glm::vec2 convert_2d(aiVector3D v)
    {
        return glm::vec2(v.x, v.y);
    }

    std::string get_name(const aiMaterial* mat, const boost::filesystem::path& package_directory)
    {
        aiString matName;
        mat->Get(AI_MATKEY_NAME, matName);
        std::string name = matName.C_Str();

        if (name == "DefaultMaterial")
            return "default";

        if (boost::starts_with(name, "//"))
            return (package_directory / name.substr(2)).string();
        if (name[0] == '/')
            return name.substr(1);
        return (package_directory / name).string();
    }

    std::string get_name(const aiMesh* mesh)
    {
        std::string name = mesh->mName.C_Str();
        if (name == "")
            return "unnamed";
        return name;
    }

    std::string get_name(const aiNode* node)
    {
        std::string name = node->mName.C_Str();
        if (name == "")
            return "unnamed";
        return name;
    }

    void convert_static_mesh(const boost::filesystem::path& package_directory, const resource::cache<graphics::material>& material_cache, const aiScene* scene, const aiMesh* src_mesh, graphics::static_mesh& dest_mesh)
    {
        std::vector<graphics::static_mesh::vertex> submesh_vertices(src_mesh->mNumVertices);
        std::vector<graphics::static_mesh::triangle> submesh_triangles(src_mesh->mNumFaces);

        for (unsigned int j = 0; j < src_mesh->mNumVertices; ++j) {
            auto pos = src_mesh->mVertices[j];
            submesh_vertices[j].position = convert_3d(pos);

            if (src_mesh->HasNormals()) {
                submesh_vertices[j].normal = convert_3d(src_mesh->mNormals[j]);
            }

            if (src_mesh->HasTangentsAndBitangents()) {
                submesh_vertices[j].tangent = convert_3d(src_mesh->mTangents[j]);
                submesh_vertices[j].bitangent = convert_3d(src_mesh->mBitangents[j]);
            }

            if (src_mesh->HasTextureCoords(0)) {
                submesh_vertices[j].texcoord = convert_2d(src_mesh->mTextureCoords[0][j]);
            }
        }

        for (unsigned int j = 0; j < src_mesh->mNumFaces; ++j) {
            aiFace f = src_mesh->mFaces[j];
            for (unsigned int k = 0; k < 3; ++k)
                submesh_triangles[j][k] = f.mIndices[k] + static_cast<unsigned int>(dest_mesh.vertices.size());
        }

        std::string material_name = get_name(scene->mMaterials[src_mesh->mMaterialIndex], package_directory);

        // TODO warn if material slot has been used.
        // TODO error if missing material.
        boost::filesystem::path material_prefix{ resource_shortname(graphics::material) };
        auto rid = resource_id_for({ material_prefix / material_name });
        dest_mesh.materials.push_back(material_cache.handle_for(rid));
        dest_mesh.material_slots.push_back(std::make_pair(dest_mesh.triangles.size(), submesh_triangles.size()));

        dest_mesh.vertices.reserve(dest_mesh.vertices.size() + submesh_vertices.size());
        dest_mesh.vertices.insert(dest_mesh.vertices.end(), submesh_vertices.begin(), submesh_vertices.end());

        dest_mesh.triangles.reserve(dest_mesh.triangles.size() + submesh_triangles.size());
        dest_mesh.triangles.insert(dest_mesh.triangles.end(), submesh_triangles.begin(), submesh_triangles.end());
    }

    void convert_static_mesh(
        const boost::filesystem::path& package_directory,
        const resource::cache<graphics::material>& material_cache,
        const aiScene* scene,
        const aiMesh* src_mesh,
        graphics::static_mesh& dest_mesh);

    template <class ContextType, class ComponentSet>
    class model_loader;

    template <class ContextType, class... Components>
    class model_loader<ContextType, type_set<Components...>> : public resource_loader<ContextType> {
    public:
        using component_set_type = component_set<Components...>;
        using blueprint_type = blueprint<component_set_type>;

        model_loader(ContextType& ctx)
            : context_{ ctx }
        {
        }

        virtual ~model_loader() = default;

        bool supports_filetype(const std::string& ext) const
        {
            static const std::set<std::string> supported_extensions{
                ".3ds", ".dae", ".fbx", ".ifc-step", ".ase", ".dxf", ".hmp", ".md2",
                ".md3", ".md5", ".mdc", ".mdl", ".nff", ".ply", ".stl", ".x", ".obj",
                ".opengex", ".smd", ".lwo", ".lxo", ".lws", ".ter", ".ac3d", ".ms3d",
                ".cob", ".q3bsp", ".xgl", ".csm", ".bvh", ".b3d", ".ndo", ".q3d",
                ".gltf", ".3mf"
            };
            return supported_extensions.count(ext) > 0;
        }

        void load(const Json::Value& global_settings, const boost::filesystem::path& source_directory, const std::string& ext, const boost::filesystem::path& source_file)
        {
            auto package_path = filesystem::make_relative(source_directory, source_file).replace_extension("");
            auto model_settings_path = source_file.parent_path() / (source_file.stem().string() + ".mdl");
            auto scene_settings_path = source_file.parent_path() / (source_file.stem().string() + ".scn");

            if (boost::filesystem::exists(scene_settings_path)) {
                load_as_blueprint(
                    package_path,
                    source_file,
                    scene_settings_path);
            } else {
                load_as_static_mesh(
                    package_path,
                    source_file,
                    model_settings_path);
            }
        }

    private:
        void convert_entity(const Json::Value& json_entity, typename blueprint_type::entity_type& blueprint_entity)
        {
            component_set_type::for_each([&](auto component_tag) {
                using component_type = typename decltype(component_tag)::type;
                if (json_entity.isMember(component_name(component_type))) {
                    component_type cmp;
                    json::from_json(json_entity[component_name(component_type)], cmp);
                    blueprint_entity.push_back(cmp);
                }
            });
        }

        void convert_node(const resource::cache<graphics::static_mesh>& static_mesh_cache,<commit_msg>Make the blueprint file extension .bpt<commit_after>#ifndef SIGMA_TOOLS_MODEL_LOADER_HPP
#define SIGMA_TOOLS_MODEL_LOADER_HPP

#include <sigma/blueprint.hpp>
#include <sigma/graphics/static_mesh.hpp>
#include <sigma/graphics/static_mesh_instance.hpp>
#include <sigma/graphics/technique.hpp>
#include <sigma/tools/packager.hpp>
#include <sigma/transform.hpp>
#include <sigma/util/filesystem.hpp>
#include <sigma/util/json_conversion.hpp>
#include <sigma/util/type_sequence.hpp>

#include <assimp/Importer.hpp>
#include <assimp/postprocess.h>
#include <assimp/scene.h>

#include <boost/algorithm/string.hpp>

#include <iostream>
#include <set>
#include <string>

namespace sigma {
namespace tools {
    glm::vec3 convert_color(aiColor3D c)
    {
        return glm::vec3(c.r, c.g, c.b);
    }

    glm::vec3 convert_3d(aiVector3D v)
    {
        return { v.x, v.y, v.z };
    }

    glm::quat convert(aiQuaternion q)
    {
        return { q.w, q.x, q.y, q.z };
    }

    glm::vec2 convert_2d(aiVector3D v)
    {
        return glm::vec2(v.x, v.y);
    }

    std::string get_name(const aiMaterial* mat, const boost::filesystem::path& package_directory)
    {
        aiString matName;
        mat->Get(AI_MATKEY_NAME, matName);
        std::string name = matName.C_Str();

        if (name == "DefaultMaterial")
            return "default";

        if (boost::starts_with(name, "//"))
            return (package_directory / name.substr(2)).string();
        if (name[0] == '/')
            return name.substr(1);
        return (package_directory / name).string();
    }

    std::string get_name(const aiMesh* mesh)
    {
        std::string name = mesh->mName.C_Str();
        if (name == "")
            return "unnamed";
        return name;
    }

    std::string get_name(const aiNode* node)
    {
        std::string name = node->mName.C_Str();
        if (name == "")
            return "unnamed";
        return name;
    }

    void convert_static_mesh(const boost::filesystem::path& package_directory, const resource::cache<graphics::material>& material_cache, const aiScene* scene, const aiMesh* src_mesh, graphics::static_mesh& dest_mesh)
    {
        std::vector<graphics::static_mesh::vertex> submesh_vertices(src_mesh->mNumVertices);
        std::vector<graphics::static_mesh::triangle> submesh_triangles(src_mesh->mNumFaces);

        for (unsigned int j = 0; j < src_mesh->mNumVertices; ++j) {
            auto pos = src_mesh->mVertices[j];
            submesh_vertices[j].position = convert_3d(pos);

            if (src_mesh->HasNormals()) {
                submesh_vertices[j].normal = convert_3d(src_mesh->mNormals[j]);
            }

            if (src_mesh->HasTangentsAndBitangents()) {
                submesh_vertices[j].tangent = convert_3d(src_mesh->mTangents[j]);
                submesh_vertices[j].bitangent = convert_3d(src_mesh->mBitangents[j]);
            }

            if (src_mesh->HasTextureCoords(0)) {
                submesh_vertices[j].texcoord = convert_2d(src_mesh->mTextureCoords[0][j]);
            }
        }

        for (unsigned int j = 0; j < src_mesh->mNumFaces; ++j) {
            aiFace f = src_mesh->mFaces[j];
            for (unsigned int k = 0; k < 3; ++k)
                submesh_triangles[j][k] = f.mIndices[k] + static_cast<unsigned int>(dest_mesh.vertices.size());
        }

        std::string material_name = get_name(scene->mMaterials[src_mesh->mMaterialIndex], package_directory);

        // TODO warn if material slot has been used.
        // TODO error if missing material.
        boost::filesystem::path material_prefix{ resource_shortname(graphics::material) };
        auto rid = resource_id_for({ material_prefix / material_name });
        dest_mesh.materials.push_back(material_cache.handle_for(rid));
        dest_mesh.material_slots.push_back(std::make_pair(dest_mesh.triangles.size(), submesh_triangles.size()));

        dest_mesh.vertices.reserve(dest_mesh.vertices.size() + submesh_vertices.size());
        dest_mesh.vertices.insert(dest_mesh.vertices.end(), submesh_vertices.begin(), submesh_vertices.end());

        dest_mesh.triangles.reserve(dest_mesh.triangles.size() + submesh_triangles.size());
        dest_mesh.triangles.insert(dest_mesh.triangles.end(), submesh_triangles.begin(), submesh_triangles.end());
    }

    void convert_static_mesh(
        const boost::filesystem::path& package_directory,
        const resource::cache<graphics::material>& material_cache,
        const aiScene* scene,
        const aiMesh* src_mesh,
        graphics::static_mesh& dest_mesh);

    template <class ContextType, class ComponentSet>
    class model_loader;

    template <class ContextType, class... Components>
    class model_loader<ContextType, type_set<Components...>> : public resource_loader<ContextType> {
    public:
        using component_set_type = component_set<Components...>;
        using blueprint_type = blueprint<component_set_type>;

        model_loader(ContextType& ctx)
            : context_{ ctx }
        {
        }

        virtual ~model_loader() = default;

        bool supports_filetype(const std::string& ext) const
        {
            static const std::set<std::string> supported_extensions{
                ".3ds", ".dae", ".fbx", ".ifc-step", ".ase", ".dxf", ".hmp", ".md2",
                ".md3", ".md5", ".mdc", ".mdl", ".nff", ".ply", ".stl", ".x", ".obj",
                ".opengex", ".smd", ".lwo", ".lxo", ".lws", ".ter", ".ac3d", ".ms3d",
                ".cob", ".q3bsp", ".xgl", ".csm", ".bvh", ".b3d", ".ndo", ".q3d",
                ".gltf", ".3mf"
            };
            return supported_extensions.count(ext) > 0;
        }

        void load(const Json::Value& global_settings, const boost::filesystem::path& source_directory, const std::string& ext, const boost::filesystem::path& source_file)
        {
            auto package_path = filesystem::make_relative(source_directory, source_file).replace_extension("");
            auto model_settings_path = source_file.parent_path() / (source_file.stem().string() + ".mdl");
            auto scene_settings_path = source_file.parent_path() / (source_file.stem().string() + ".bpt");

            if (boost::filesystem::exists(scene_settings_path)) {
                load_as_blueprint(
                    package_path,
                    source_file,
                    scene_settings_path);
            } else {
                load_as_static_mesh(
                    package_path,
                    source_file,
                    model_settings_path);
            }
        }

    private:
        void convert_entity(const Json::Value& json_entity, typename blueprint_type::entity_type& blueprint_entity)
        {
            component_set_type::for_each([&](auto component_tag) {
                using component_type = typename decltype(component_tag)::type;
                if (json_entity.isMember(component_name(component_type))) {
                    component_type cmp;
                    json::from_json(json_entity[component_name(component_type)], cmp);
                    blueprint_entity.push_back(cmp);
                }
            });
        }

        void convert_node(const resource::cache<graphics::static_mesh>& static_mesh_cache,
{'fraction_non_alphanumeric': 0.13490039291376577, 'fraction_numerical': 0.004825256772592541, 'mean_word_length': 2.6008935219657485, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4537001', 'n_tokens_mistral': 4575, 'n_tokens_neox': 4432, 'n_words': 782}