<commit_before>// Copyright (C) 2017 <NAME>
// This file is part of the "Nazara Engine - Core module"
// For conditions of distribution and use, see copyright notice in Config.hpp

#include <Nazara/Core/StringExt.hpp>
#include <Nazara/Core/Algorithm.hpp>
#include <Utfcpp/utf8.h>
#include <Nazara/Core/Debug.hpp>

namespace Nz
{
	namespace
	{
		bool IsSpace(char32_t character)
		{
			return character == '\t' || Unicode::GetCategory(character) & Unicode::Category_Separator;
		}

		char ToLower(char character)
		{
			if (character >= 'A' && character <= 'Z')
				return character + ('a' - 'A');
			else
				return character;
		}

		char ToUpper(char character)
		{
			if (character >= 'a' && character <= 'z')
				return character + ('A' - 'a');
			else
				return character;
		}

		template<std::size_t S>
		struct WideConverter
		{
			static std::string From(const wchar_t* wstr, std::size_t size)
			{
				if constexpr (S == 2)
				{
					// UTF-16 (Windows)
					return FromUtf16String(std::u16string_view(reinterpret_cast<const char16_t*>(wstr), size));
				}
				else if constexpr (S == 4)
				{
					// UTF-32 (Linux)
					return FromUtf32String(std::u32string_view(reinterpret_cast<const char32_t*>(wstr), size));
				}
				else
				{
					static_assert(AlwaysFalse<S>, "Unsupported platform");
					return std::string("<platform error>");
				}
			}

			static std::wstring To(const std::string_view& str)
			{
				if constexpr (S == 2)
				{
					std::wstring result;
					utf8::utf8to16(str.begin(), str.end(), std::back_inserter(result));

					return result;
				}
				else if constexpr (S == 4)
				{
					std::wstring result;
					utf8::utf8to32(str.begin(), str.end(), std::back_inserter(result));

					return result;
				}
				else
				{
					static_assert(AlwaysFalse<S>, "Unsupported platform");
					return std::string("<platform error>");
				}
			}
		};
	}

	std::string FromUtf16String(const char16_t* u16str)
	{
		std::size_t size = std::char_traits<char16_t>::length(u16str);
		return FromUtf16String(std::u16string_view(u16str, size));
	}

	std::string FromUtf16String(const std::u16string_view& u16str)
	{
		std::string result;
		utf8::utf16to8(u16str.begin(), u16str.end(), std::back_inserter(result));

		return result;
	}

	std::string FromUtf32String(const char32_t* u32str)
	{
		std::size_t size = std::char_traits<char32_t>::length(u32str);<commit_msg>StringExt: Fix some template mistakes<commit_after>// Copyright (C) 2017 <NAME>
// This file is part of the "Nazara Engine - Core module"
// For conditions of distribution and use, see copyright notice in Config.hpp

#include <Nazara/Core/StringExt.hpp>
#include <Nazara/Core/Algorithm.hpp>
#include <Utfcpp/utf8.h>
#include <Nazara/Core/Debug.hpp>

namespace Nz
{
	namespace
	{
		bool IsSpace(char32_t character)
		{
			return character == '\t' || Unicode::GetCategory(character) & Unicode::Category_Separator;
		}

		char ToLower(char character)
		{
			if (character >= 'A' && character <= 'Z')
				return character + ('a' - 'A');
			else
				return character;
		}

		char ToUpper(char character)
		{
			if (character >= 'a' && character <= 'z')
				return character + ('A' - 'a');
			else
				return character;
		}

		template<std::size_t S>
		struct WideConverter
		{
			static std::string From(const wchar_t* wstr, std::size_t size)
			{
				if constexpr (S == 2)
				{
					// UTF-16 (Windows)
					return FromUtf16String(std::u16string_view(reinterpret_cast<const char16_t*>(wstr), size));
				}
				else if constexpr (S == 4)
				{
					// UTF-32 (Linux)
					return FromUtf32String(std::u32string_view(reinterpret_cast<const char32_t*>(wstr), size));
				}
				else
				{
					static_assert(AlwaysFalse<std::integral_constant<std::size_t, S>>::value, "Unsupported platform");
					return std::string("<platform error>");
				}
			}

			static std::wstring To(const std::string_view& str)
			{
				if constexpr (S == 2)
				{
					std::wstring result;
					utf8::utf8to16(str.begin(), str.end(), std::back_inserter(result));

					return result;
				}
				else if constexpr (S == 4)
				{
					std::wstring result;
					utf8::utf8to32(str.begin(), str.end(), std::back_inserter(result));

					return result;
				}
				else
				{
					static_assert(AlwaysFalse<std::integral_constant<std::size_t, S>>, "Unsupported platform");
					return std::wstring(L"<platform error>");
				}
			}
		};
	}

	std::string FromUtf16String(const char16_t* u16str)
	{
		std::size_t size = std::char_traits<char16_t>::length(u16str);
		return FromUtf16String(std::u16string_view(u16str, size));
	}

	std::string FromUtf16String(const std::u16string_view& u16str)
	{
		std::string result;
		utf8::utf16to8(u16str.begin(), u16str.end(), std::back_inserter(result));

		return result;
	}

	std::string FromUtf32String(const char32_t* u32str)
	{
		std::size_t size = std::char_traits<char32_t>::length(u32str);
{'fraction_non_alphanumeric': 0.14929346713086217, 'fraction_numerical': 0.030718820397296745, 'mean_word_length': 3.6826462128475552, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2242585', 'n_tokens_mistral': 2171, 'n_tokens_neox': 1724, 'n_words': 359}