<commit_before>#ifndef GUARD_MULTISTRING_H
#define GUARD_MULTISTRING_H 1

#include <string>

struct multistring {
	multistring(const char * _ns, const wchar_t * _ws,
	            const char16_t * _s16, const char32_t * _s32)
	: ns(_ns), ws(_ws), s16(_s16), s32(_s32)
	{ }

	template <typename T>
	const std::basic_string<T> & get() const;

	std::string    ns;
	std::wstring   ws;
	std::u16string s16;
	std::u32string s32;
};

#define TRIPLE_CAT_(a_, b_, c_) a_ ## b_ ## c_
#define TRIPLE_CAT(a_, b_, c_) TRIPLE_CAT_(a_, b_, c_)
#define NARROW(s_) TRIPLE_CAT(, s_, )
#define WIDE(s_) TRIPLE_CAT(L, s_, )
#define UTF8(s_) TRIPLE_CAT(u8, s_, )
#define UTF16(s_) TRIPLE_CAT(u, s_, )
#define UTF32(s_) TRIPLE_CAT(U, s_, )

#define DEF_MULTISTRING(name, literalval) \
	multistring name(NARROW(literalval), WIDE(literalval), \
	                 UTF16(literalval), UTF32(literalval))


#endif // GUARD_MULTISTRING_H<commit_msg>Added more stuff here - ability to use arbitrary data in string (nul chars)<commit_after>#ifndef GUARD_MULTISTRING_H
#define GUARD_MULTISTRING_H 1

#include <string>

struct multistring {
	multistring(const char * _ns, const wchar_t * _ws,
	            const char16_t * _s16, const char32_t * _s32)
	: ns(_ns), ws(_ws), s16(_s16), s32(_s32)
	{ }

	multistring(const char * _ns, size_t _ns_len,
	            const wchar_t * _ws, size_t _ws_len,
	            const char16_t * _s16, size_t _s16_len,
	            const char32_t * _s32, size_t _s32_len)
	: ns(_ns, _ns_len)
	, ws(_ws, _ws_len)
	, s16(_s16, _s16_len)
	, s32(_s32, _s32_len)
	{ }

	template <typename T>
	const std::basic_string<T> & get() const;

	std::string    ns;
	std::wstring   ws;
	std::u16string s16;
	std::u32string s32;
};

#define TRIPLE_CAT_(a_, b_, c_) a_ ## b_ ## c_
#define TRIPLE_CAT(a_, b_, c_) TRIPLE_CAT_(a_, b_, c_)
#define NARROW(s_) TRIPLE_CAT(, s_, )
#define WIDE(s_) TRIPLE_CAT(L, s_, )
#define UTF8(s_) TRIPLE_CAT(u8, s_, )
#define UTF16(s_) TRIPLE_CAT(u, s_, )
#define UTF32(s_) TRIPLE_CAT(U, s_, )

#define DEF_MULTISTRING(name, literalval) \
	multistring name(NARROW(literalval), \
	                 sizeof(NARROW(literalval)) / sizeof(char), \
	                 WIDE(literalval), \
	                 sizeof(WIDE(literalval)) / sizeof(wchar_t), \
	                 UTF16(literalval), \
	                 sizeof(UTF16(literalval)) / sizeof(char16_t), \
	                 UTF32(literalval), \
	                 sizeof(UTF32(literalval)) / sizeof(char32_t))

template<>
inline const std::basic_string<char> & multistring::get<char>() const
{ return ns; }

template<>
inline const std::basic_string<wchar_t> & multistring::get<wchar_t>() const
{ return ws; }

template<>
inline const std::basic_string<char16_t> & multistring::get<char16_t>() const
{ return s16; }

template<>
inline const std::basic_string<char32_t> & multistring::get<char32_t>() const
{ return s32; }

template <typename T>
const std::basic_string<char> byte_oriented_string(bool little_endian);

#endif // GUARD_MULTISTRING_H
{'fraction_non_alphanumeric': 0.13566666666666666, 'fraction_numerical': 0.038, 'mean_word_length': 3.90359477124183, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083774', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1189, 'n_words': 254}