<commit_before>//
// Copyright (C) <NAME>. 2018
// Distributed under the MIT License (license terms are at http://opensource.org/licenses/MIT).
//

#include "stdafx.h"

#include "Render/DX12/Shader/ShaderPreprocessorDX12.h"

#include "AssetsSystem/AssetsSystem.h"

namespace Kioto::Renderer
{
namespace ShaderPreprocessorDX12
{

std::vector<std::string> m_preprocessedHeaders;
static constexpr uint16 m_maxDepth = 128;

std::string UnfoldIncludes(std::string source, uint16 recursionDepth)
{
    if (recursionDepth > m_maxDepth)
    {
        throw "Too deep shader includes.";
        return "";
    }
    size_t includePos = source.find("#include", 0);
    size_t includeFin = source.find("\n", includePos);
    while (includePos != std::string::npos)
    {
        if (includeFin != std::string::npos)
        {
            std::string s = source.substr(includePos, includeFin - includePos);
            source.erase(includePos, includeFin - includePos);

            std::string relativeIncludePath;
            bool startFound = false;
            for (int i = 8; i < s.size(); ++i)
            {
                if (s[i] == '\"' && !startFound)
                {
                    startFound = true;
                    continue;
                }
                else if (!startFound)
                    continue;
                else if (s[i] == '\"')
                    break;
                relativeIncludePath += s[i];
            }
            relativeIncludePath.insert(0, "Shaders\\");

            auto it = std::find(m_preprocessedHeaders.begin(), m_preprocessedHeaders.end(), relativeIncludePath);
            if (it == m_preprocessedHeaders.end())
            {
                std::wstring shaderPath = AssetsSystem::GetAssetFullPath(StrToWstr(relativeIncludePath));
                bool isExist = AssetsSystem::CheckIfFileExist(shaderPath);
                std::string incl = AssetsSystem::ReadFileAsString(std::string(shaderPath.begin(), shaderPath.end()));
                source.insert(includePos, UnfoldIncludes(incl, recursionDepth + 1));
                m_preprocessedHeaders.push_back(relativeIncludePath);
            }
        }

        includePos = source.find("#include", includePos);
        includeFin = source.find("\n", includePos);
    }

    return source;
}

ParseResult ParseShader(const std::string& path)
{
    ParseResult res;
    m_preprocessedHeaders.clear();
    std::string shaderStr = AssetsSystem::ReadFileAsString(path);
    shaderStr = ShaderPreprocessorDX12::UnfoldIncludes(shaderStr, 0);
    res.output = shaderStr;
    return res;
}

}
}<commit_msg>Vertex layout parser.<commit_after>//
// Copyright (C) <NAME>. 2018
// Distributed under the MIT License (license terms are at http://opensource.org/licenses/MIT).
//

#include "stdafx.h"

#include "Render/DX12/Shader/ShaderPreprocessorDX12.h"
#include "Render/VertexLayout.h"

#include "AssetsSystem/AssetsSystem.h"

namespace Kioto::Renderer
{
namespace ShaderPreprocessorDX12
{

std::vector<std::string> m_preprocessedHeaders;
static constexpr uint16 m_maxDepth = 128;

std::string UnfoldIncludes(std::string source, uint16 recursionDepth)
{
    if (recursionDepth > m_maxDepth)
    {
        throw "Too deep shader includes.";
        return "";
    }
    size_t includePos = source.find("#include", 0);
    size_t includeFin = source.find("\n", includePos);
    while (includePos != std::string::npos)
    {
        if (includeFin != std::string::npos)
        {
            std::string s = source.substr(includePos, includeFin - includePos);
            source.erase(includePos, includeFin - includePos);

            std::string relativeIncludePath;
            bool startFound = false;
            for (int i = 8; i < s.size(); ++i)
            {
                if (s[i] == '\"' && !startFound)
                {
                    startFound = true;
                    continue;
                }
                else if (!startFound)
                    continue;
                else if (s[i] == '\"')
                    break;
                relativeIncludePath += s[i];
            }
            relativeIncludePath.insert(0, "Shaders\\");

            auto it = std::find(m_preprocessedHeaders.begin(), m_preprocessedHeaders.end(), relativeIncludePath);
            if (it == m_preprocessedHeaders.end())
            {
                std::wstring shaderPath = AssetsSystem::GetAssetFullPath(StrToWstr(relativeIncludePath));
                bool isExist = AssetsSystem::CheckIfFileExist(shaderPath);
                std::string incl = AssetsSystem::ReadFileAsString(std::string(shaderPath.begin(), shaderPath.end()));
                source.insert(includePos, UnfoldIncludes(incl, recursionDepth + 1));
                m_preprocessedHeaders.push_back(relativeIncludePath);
            }
        }

        includePos = source.find("#include", includePos);
        includeFin = source.find("\n", includePos);
    }

    return source;
}

bool IsEmptyChar(char c)
{
    return c == '\n' || c == ' ' || c == '\r\n';
}

uint8 CharToInt(char c)
{
    return c - '0';
}

bool TryParseSemantics(const std::string& source, size_t pos, eVertexSemantic& semantic, uint8& semanticIndex, uint8& semStringLen)
{
    if (source.substr(pos, 8) == "POSITION")
    {
        semantic = eVertexSemantic::Position;
        if (isdigit(source[pos + 8]))
        {
            semanticIndex = CharToInt(source[pos + 8]);
            semStringLen = 9;
            return true;
        }
        semanticIndex = 0;
        semStringLen = 8;
        return true;
    }
    if (source.substr(pos, 6) == "NORMAL")
    {
        semantic = eVertexSemantic::Normal;
        if (isdigit(source[pos + 6]))
        {
            semanticIndex = CharToInt(source[pos + 6]);
            semStringLen = 7;
            return true;
        }
        semanticIndex = 0;
        semStringLen = 6;
        return true;
    }
    if (source.substr(pos, 8) == "TEXCOORD")
    {
        semantic = eVertexSemantic::Texcoord;
        if (isdigit(source[pos + 8]))
        {
            semanticIndex = CharToInt(source[pos + 8]);
            semStringLen = 9;
            return true;
        }
        semanticIndex = 0;
        semStringLen = 8;
        return true;
    }
    if (source.substr(pos, 5) == "COLOR")
    {
        semantic = eVertexSemantic::Color;
        if (isdigit(source[pos + 5]))
        {
            semanticIndex = CharToInt(source[pos + 5]);
            semStringLen = 6;
            return true;
        }
        semanticIndex = 0;
        semStringLen = 6;
        return true;
    }
    if (source.substr(pos, 9) == "BITANGENT")
    {
        semantic = eVertexSemantic::Bitangent;
        if (isdigit(source[pos + 9]))
        {
            semanticIndex = CharToInt(source[pos + 9]);
            semStringLen = 10;
            return true;
        }
        semanticIndex = 0;
        semStringLen = 9;
        return true;
    }
    if (source.substr(pos, 7) == "TANGENT")
    {
        semantic = eVertexSemantic::Tangent;
        if (isdigit(source[pos + 7]))
        {
            semanticIndex = CharToInt(source[pos + 7]);
            semStringLen = 8;
            return true;
        }
        semanticIndex = 0;
        semStringLen = 7;
        return true;
    }
    return false;
}

VertexLayout GetVertexLayout(const std::string& source)
{
    static constexpr uint8 floatLen = 5;

    VertexLayout res;
    size_t structBegin = source.find("struct vIn", 0);
    if (structBegin == std::string::npos)
        throw "Input structure does not exist";
    size_t closeBracetPos = structBegin;
    for (; closeBracetPos < source.size() - 1; ++closeBracetPos)
    {
        if (source[closeBracetPos] == '}' && source[closeBracetPos + 1] == ';')
            break;
    }
    for (size_t i = structBegin; i < closeBracetPos - 6; ++i)
    {
        eVertexDataFormat format;
        eVertexSemantic semantic;
        byte semanticIndex;
        if (source.substr(i, 5) == "float")
        {
            bool typeFound = false;
            if (IsEmptyChar(source[i + floatLen]))
            {
                format = eVertexDataFormat::R32;
                typeFound = true;
            }
            else if (isdigit(source[i + floatLen]) && IsEmptyChar(source[i + floatLen + 1]))
            {
                uint8 dim = CharToInt(source[i + floatLen]);
                if (dim == 2)
                    format = eVertexDataFormat::R32_G32;
                else if (dim == 3)
                    format = eVertexDataFormat::R32_G32_B32;
                else if (dim == 4)
                    format = eVertexDataFormat::R32_G32_B32_A32;
                else
                    throw "wtf?";

                typeFound = true;
            }
            if (typeFound)
            {
                bool foundSemStart = false;
                for (size_t j = i + floatLen; j < closeBracetPos; ++j)
                {
                    if (source[j] == ':')
                        foundSemStart = true;
                    if (source[j] == ';')
                    {
                        if (!foundSemStart)
                            throw "wtf?";
                        break;
                    }
                    uint8 semStingLen = 0;
                    if (foundSemStart && TryParseSemantics(source, j, semantic, semanticIndex, semStingLen))
                    {
                        res.AddElement(semantic, semanticIndex, format);
                        i = j + semStingLen;
                        break;
                    }
                }
            }
        }
    }
    return res;
}

ParseResult ParseShader(const std::string& path)
{
    ParseResult res;
    m_preprocessedHeaders.clear();
    std::string shaderStr = AssetsSystem::ReadFileAsString(path);
    shaderStr = ShaderPreprocessorDX12::UnfoldIncludes(shaderStr, 0);
    VertexLayout vl = GetVertexLayout(shaderStr);
    res.output = shaderStr;
    return res;
}

}
}
{'fraction_non_alphanumeric': 0.11632796075683252, 'fraction_numerical': 0.012313544899389328, 'mean_word_length': 1.8316326530612246, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4262996', 'n_tokens_mistral': 3055, 'n_tokens_neox': 2844, 'n_words': 602}