<commit_before>#include <cstring>

#ifndef TEXT_ENCODER
#define TEXT_ENCODER

namespace lyrics
{
	class TextEncoder
	{
	public:
		char16_t *EncodeToUTF16( const unsigned char * const data, const unsigned int size, unsigned int &length )
		{
			using std::size_t;
			using std::memcpy;

			if ( data == nullptr )
			{
				return nullptr;
			}

			switch ( data[0] )
			{
			case 0xEF:
				if ( data[1] == 0xBB && data[2] == 0xBF )
				{
					return UTF8ToUTF16( data + 3, size - 3, length );
				}
				else
				{
					return UTF8ToUTF16( data, size, length );
				}
				break;

			case 0xFE:
				if ( data[1] == 0xFF )
				{
					// TODO: UTF-16 big endian
				}
				else
				{
					return UTF8ToUTF16( data, size, length );
				}
				break;

			case 0xFF:
				if ( data[1] == 0xFE )
				{
					if ( data[2] != 0x00 || data[3] != 0x00 )
					{
						char16_t *tStr;
						size_t tLength;

						for ( tLength = 1; ( ( char16_t * )data )[tLength] != 0 ; tLength++ );
						tStr = new char16_t [tLength];
						memcpy( tStr, data + 2, tLength );
						length = tLength;

						return tStr;
					}
					else
					{
						return UTF32ToUTF16( ( char32_t * )( data + 4 ), ( size - 4 ) >> 2, length );
					}
				}
				else
				{
					return UTF8ToUTF16( data, size, length );
				}
				break;

			case 0x00:
				if ( data[1] == 0x00 && data[2] == 0xFE && data[3] == 0xFF )
				{
					// TODO: UTF-32 big endian
				}
				else
				{
					return UTF8ToUTF16( data, size, length );<commit_msg>Copy size of bytes UTF-16LE data<commit_after>#include <cstring>

#ifndef TEXT_ENCODER
#define TEXT_ENCODER

namespace lyrics
{
	class TextEncoder
	{
	public:
		char16_t *EncodeToUTF16( const unsigned char * const data, const unsigned int size, unsigned int &length )
		{
			using std::size_t;
			using std::memcpy;

			if ( data == nullptr )
			{
				return nullptr;
			}

			switch ( data[0] )
			{
			case 0xEF:
				if ( data[1] == 0xBB && data[2] == 0xBF )
				{
					return UTF8ToUTF16( data + 3, size - 3, length );
				}
				else
				{
					return UTF8ToUTF16( data, size, length );
				}
				break;

			case 0xFE:
				if ( data[1] == 0xFF )
				{
					// TODO: UTF-16 big endian
				}
				else
				{
					return UTF8ToUTF16( data, size, length );
				}
				break;

			case 0xFF:
				if ( data[1] == 0xFE )
				{
					if ( data[2] != 0x00 || data[3] != 0x00 )
					{
						char16_t *tStr;

						for ( length = 1; ( ( char16_t * )data )[length] != 0; length++ );
						tStr = new char16_t [length];
						memcpy( tStr, data + 2, size );

						return tStr;
					}
					else
					{
						return UTF32ToUTF16( ( char32_t * )( data + 4 ), ( size - 4 ) >> 2, length );
					}
				}
				else
				{
					return UTF8ToUTF16( data, size, length );
				}
				break;

			case 0x00:
				if ( data[1] == 0x00 && data[2] == 0xFE && data[3] == 0xFF )
				{
					// TODO: UTF-32 big endian
				}
				else
				{
					return UTF8ToUTF16( data, size, length );
{'fraction_non_alphanumeric': 0.13637913399249915, 'fraction_numerical': 0.05114217524718718, 'mean_word_length': 1.925224327018943, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1835695', 'n_tokens_mistral': 1597, 'n_tokens_neox': 1110, 'n_words': 289}