<commit_before>/*******************************************************************************
* gtparser - Generic Text parsing functions library
* Copyright (C) 2008-2017 <NAME>, https://github.com/mbuilov/gtparser
* Licensed under LGPL version 2.1 or any later version, see COPYING
*******************************************************************************/

/* cstring_parser.c */

#include "gtparser/gtparser_config.h"
#include "gtparser/cstring_parser.h"
#include "gtparser/parser_base.h"
#include "gtparser/name_scanner.h"

static enum PARSE_CSTRING_ERR _parse_cstring(unsigned *line, const char **current,
	unsigned *back_column, size_t *removed, const char *const end)
{
	const char quote = _src_iter_get_char(*current);
	while (_src_iter_next(current, end)) {
		char c = _src_iter_get_char(*current);
_switch_c:
		if (quote == c)
			return PARSE_CSTRING_OK; /* it points to the last quote */
		if ('\n' == c || '\r' == c)
			return PARSE_CSTRING_UNESCAPED_NEWLINE; /* unescaped line-feed or carridge-return */
		if ('\\' == c) {
			if (!_src_iter_next(current, end))
				break; /* error: unterminated string */
			(*removed)++; /* '\' must be removed */
			c = _src_iter_get_char(*current);
			if ('\r' == c) {
				if (!_src_iter_next(current, end))
					break; /* error: unterminated string */
				c = _src_iter_get_char(*current);
				if ('\n' != c)
					return PARSE_CSTRING_EXPECTING_LINE_FEED; /* expecting line-feed after carridge-return */
				(*removed) += 2; /* line continuation (split) must be removed */
			}
			else if ('\n' == c)
				(*removed)++; /* line continuation (split) must be removed */
			else if ('x' == c) {
				/* hexadecimal constant */
				if (!_src_iter_next(current, end))
					break; /* error: unterminated string */
				c = _src_iter_get_char(*current);
				{
					unsigned n = _hex_char_value(c);
					if (n > 15)
						return PARSE_CSTRING_EXPECTING_HEX_DIGIT; /* expecting hexadecimal digit in hex escape sequence after \x */
					if (!_src_iter_next(current, end))
						break; /* error: unterminated string */
					c = _src_iter_get_char(*current);
					{
						unsigned m = _hex_char_value(c);
						if (m <= 15 && n + m) {
							(*removed) += 2; /* xf in "\xff" */
							continue;
						}
					}
					if (n) {
						(*removed)++; /* x in "\xf" */
						goto _switch_c;
					}
					(*current)--; /* points to 0 in "\x0?" */
					c = '\0'; /* error: null inside string */
				}
			}
			else {
				unsigned n = digit_value(c);
				if (n <= 7) {
					/* octal constant */
					if (!_src_iter_next(current, end))
						break; /* error: unterminated string */
					c = _src_iter_get_char(*current);
					{
						unsigned m = digit_value(c);
						if (m <= 7) {
							if (!_src_iter_next(current, end))
								break; /* error: unterminated string */
							n = n*8 + m;
							c = _src_iter_get_char(*current);
							m = digit_value(c);
							if (m <= 7) {
								if (n*8 + m > 255) {
									(*current) -= 2; /* points to 777 in "\777" */
									return PARSE_CSTRING_TOO_BIG_OCTAL; /* too big octal character value > 255 in string */
								}
								if (n + m) {
									(*removed) += 2; /* 12 in "\123" */
									continue;
								}
							}
							else if (n) {
								(*removed)++; /* 1 in "\12" */
								goto _switch_c;
							}
							(*current)--; /* error: null inside string */
						}
						else if (n)
							goto _switch_c;
					}<commit_msg>rename: _src_iter_get_char() -> _src_iter_current_char()<commit_after>/*******************************************************************************
* gtparser - Generic Text parsing functions library
* Copyright (C) 2008-2017 <NAME>, https://github.com/mbuilov/gtparser
* Licensed under LGPL version 2.1 or any later version, see COPYING
*******************************************************************************/

/* cstring_parser.c */

#include "gtparser/gtparser_config.h"
#include "gtparser/cstring_parser.h"
#include "gtparser/parser_base.h"
#include "gtparser/name_scanner.h"

static enum PARSE_CSTRING_ERR _parse_cstring(unsigned *line, const char **current,
	unsigned *back_column, size_t *removed, const char *const end)
{
	const char quote = _src_iter_current_char(*current);
	while (_src_iter_next(current, end)) {
		char c = _src_iter_current_char(*current);
_switch_c:
		if (quote == c)
			return PARSE_CSTRING_OK; /* it points to the last quote */
		if ('\n' == c || '\r' == c)
			return PARSE_CSTRING_UNESCAPED_NEWLINE; /* unescaped line-feed or carridge-return */
		if ('\\' == c) {
			if (!_src_iter_next(current, end))
				break; /* error: unterminated string */
			(*removed)++; /* '\' must be removed */
			c = _src_iter_current_char(*current);
			if ('\r' == c) {
				if (!_src_iter_next(current, end))
					break; /* error: unterminated string */
				c = _src_iter_current_char(*current);
				if ('\n' != c)
					return PARSE_CSTRING_EXPECTING_LINE_FEED; /* expecting line-feed after carridge-return */
				(*removed) += 2; /* line continuation (split) must be removed */
			}
			else if ('\n' == c)
				(*removed)++; /* line continuation (split) must be removed */
			else if ('x' == c) {
				/* hexadecimal constant */
				if (!_src_iter_next(current, end))
					break; /* error: unterminated string */
				c = _src_iter_current_char(*current);
				{
					unsigned n = _hex_char_value(c);
					if (n > 15)
						return PARSE_CSTRING_EXPECTING_HEX_DIGIT; /* expecting hexadecimal digit in hex escape sequence after \x */
					if (!_src_iter_next(current, end))
						break; /* error: unterminated string */
					c = _src_iter_current_char(*current);
					{
						unsigned m = _hex_char_value(c);
						if (m <= 15 && n + m) {
							(*removed) += 2; /* xf in "\xff" */
							continue;
						}
					}
					if (n) {
						(*removed)++; /* x in "\xf" */
						goto _switch_c;
					}
					(*current)--; /* points to 0 in "\x0?" */
					c = '\0'; /* error: null inside string */
				}
			}
			else {
				unsigned n = digit_value(c);
				if (n <= 7) {
					/* octal constant */
					if (!_src_iter_next(current, end))
						break; /* error: unterminated string */
					c = _src_iter_current_char(*current);
					{
						unsigned m = digit_value(c);
						if (m <= 7) {
							if (!_src_iter_next(current, end))
								break; /* error: unterminated string */
							n = n*8 + m;
							c = _src_iter_current_char(*current);
							m = digit_value(c);
							if (m <= 7) {
								if (n*8 + m > 255) {
									(*current) -= 2; /* points to 777 in "\777" */
									return PARSE_CSTRING_TOO_BIG_OCTAL; /* too big octal character value > 255 in string */
								}
								if (n + m) {
									(*removed) += 2; /* 12 in "\123" */
									continue;
								}
							}
							else if (n) {
								(*removed)++; /* 1 in "\12" */
								goto _switch_c;
							}
							(*current)--; /* error: null inside string */
						}
						else if (n)
							goto _switch_c;
					}
{'fraction_non_alphanumeric': 0.19155703339598093, 'fraction_numerical': 0.013300563828249241, 'mean_word_length': 2.60500260552371, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3786753', 'n_tokens_mistral': 3079, 'n_tokens_neox': 2296, 'n_words': 604}