<commit_before>// ================================================================================
// ==      This file is a part of Turbo Badger. (C) 2011-2014, <NAME>      ==
// ==                     See tb_core.h for more information.                    ==
// ================================================================================

#include "parser/tb_parser.h"
#include "tb_tempbuffer.h"
#include <assert.h>

namespace tb {

// == Util functions ====================================================================

void UnescapeString(char *str)
{
	char *dst = str, *src = str;
	while (*src)
	{
		if (*src == '\\')
		{
			bool code_found = true;
			if (src[1] == 'n')
				*dst = '\n';
			else if (src[1] == 'r')
				*dst = '\r';
			else if (src[1] == 't')
				*dst = '\t';
			else if (src[1] == '\"')
				*dst = '\"';
			else if (src[1] == '\'')
				*dst = '\'';
			else if (src[1] == '\\')
				*dst = '\\';
			else
				code_found = false;
			if (code_found)
			{
				src += 2;
				dst++;
				continue;
			}
		}
		*dst = *src;
		dst++;
		src++;
	}
	*dst = 0;
}

bool is_white_space(const char *str)
{
	switch (*str)
	{
	case ' ':
	case '\t':
		return true;
	default:
		return false;
	}
}

/** Return true if the given string starts with a color.
	Ex: #ffdd00, #fd0 */
bool is_start_of_color(const char *str)
{
	if (*str++ != '#')
		return false;
	int digit_count = 0;
	while ((*str >= '0' && *str <= '9') ||
			(*str >= 'a' && *str <= 'f') ||
			(*str >= 'A' && *str <= 'F'))
	{
		str++;
		digit_count++;
	}
	return digit_count == 8 || digit_count == 6 || digit_count == 4 || digit_count == 3;
}

/** Return true if the given string may be a node reference, such
	as language strings or TBNodeRefTree references. */
bool is_start_of_reference(const char *str)<commit_msg>Handle more escape codes when parsing strings.<commit_after>// ================================================================================
// ==      This file is a part of Turbo Badger. (C) 2011-2014, <NAME>      ==
// ==                     See tb_core.h for more information.                    ==
// ================================================================================

#include "parser/tb_parser.h"
#include "tb_tempbuffer.h"
#include "utf8/utf8.h"
#include <assert.h>
#include <ctype.h>

namespace tb {

// == Util functions ====================================================================

static bool is_hex(char c)
{
	return ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'));
}

static uint32 parse_hex(char *&src, int max_count)
{
	uint32 hex = 0;
	for (int i = 0; i < max_count; i++)
	{
		char c = *src;
		if (!is_hex(c))
			break;
		hex <<= 4;
		hex |= isdigit(c) ? c - '0' : tolower(c) - 'a' + 10;
		src++;
	}
	return hex;
}

void UnescapeString(char *str)
{
	// fast forward to any escape sequence
	while (*str && *str != '\\')
		str++;

	char *dst = str, *src = str;
	while (*src)
	{
		if (*src == '\\')
		{
			bool code_found = true;
			switch (src[1])
			{
			case 'a': *dst = '\a'; break;
			case 'b': *dst = '\b'; break;
			case 'f': *dst = '\f'; break;
			case 'n': *dst = '\n'; break;
			case 'r': *dst = '\r'; break;
			case 't': *dst = '\t'; break;
			case 'v': *dst = '\v'; break;
			case '0': *dst = '\0'; break;
			case '\"': *dst = '\"'; break;
			case '\'': *dst = '\''; break;
			case '\\': *dst = '\\'; break;
			case 'x': // \xXX
			case 'u': // \uXXXX
			{
				// This should be safe. A utf-8 character can be at most 4 bytes,
				// and we have 4 bytes to use for \xXX and 6 for \uXXXX.
				src += 2;
				if (UCS4 hex = parse_hex(src, src[1] == 'x' ? 2 : 4))
					dst += utf8::encode(hex, dst);
				continue;
			}
			default:
				code_found = false;
			}
			if (code_found)
			{
				src += 2;
				dst++;
				continue;
			}
		}
		*dst = *src;
		dst++;
		src++;
	}
	*dst = 0;
}

bool is_white_space(const char *str)
{
	switch (*str)
	{
	case ' ':
	case '\t':
		return true;
	default:
		return false;
	}
}

/** Return true if the given string starts with a color.
	Ex: #ffdd00, #fd0 */
bool is_start_of_color(const char *str)
{
	if (*str++ != '#')
		return false;
	int digit_count = 0;
	while (is_hex(*str))
	{
		str++;
		digit_count++;
	}
	return digit_count == 8 || digit_count == 6 || digit_count == 4 || digit_count == 3;
}

/** Return true if the given string may be a node reference, such
	as language strings or TBNodeRefTree references. */
bool is_start_of_reference(const char *str)
{'fraction_non_alphanumeric': 0.2845601436265709, 'fraction_numerical': 0.015933572710951527, 'mean_word_length': 2.7707275803722506, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '331613', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1577, 'n_words': 467}