<commit_before>#include "jsondefs.h"
#include "jsonparser.h"


namespace LightSpeed {

double strtod(const char *string, char **endPtr);

namespace JSON {



template<typename T>
Value Parser<T>::parse() {

		while (true) {
			char x = iter.getNext();
			switch (x) {
				case '"': return parseString();
				case '{': return parseObject(factory->object());
				case '[': return parseArray(factory->array());
				case 't': parseCheck(strTrue+1);
						  if (sharedTrue == null) sharedTrue = factory->newValue(true);
						  return sharedTrue;
				case 'f': parseCheck(strFalse+1);
						  if (sharedFalse == null) sharedFalse = factory->newValue(false);
						  return sharedFalse;
				case 'n': parseCheck(strNull+1);
						  if (sharedNull == null) sharedNull = factory->newNullNode();
						  return sharedNull;
				case ' ':
				case '\t':
				case '\r':
				case '\n':
				case '\b':
				case '\a':
				case '\0': break;
				default: return parseValue(x);
			}
		}
	}


template<typename T>
PNode Parser<T>::parseString() {
	parseRawString();
	if (strBuff.empty()) {
		if (sharedEmptyStr == null) sharedEmptyStr = factory->newValue(ConstStrA());
		return sharedEmptyStr;
	} else {
		return factory->newValue(ConstStrA(strBuff));
	}
}


template<typename T>
void Parser<T>::parseRawString() {
	char x = iter.getNext();
		strBuff.clear();
		while (x != '"') {
			if (x != '\\') {
				strBuff.add(x);
			} else {
				x = iter.getNext();
				char y;

				switch (x) {
					case '"': y = x;break;
					case '\\':y = x;break;
					case '/': y = x ;break;
					case 'b': y  = '\b';break;
					case 'f': y = '\f';break;
					case 'n': y = '\n'; break;
					case 'r': y = '\r';break;
					case 't': y = '\t';break;
					case 'u': {
							char buff[5];
							buff[4] = 0;
							for (int i = 0; i < 4; i++) {
								char z  = iter.getNext();
								if ((z >= '0' && z <= '9')||(z >='A' && z <= 'F') || (z >= 'a' && z <= 'f')) {
									buff[i] = (char)z;
								} else {
									throw ParseError_t(THISLOCATION,strBuff);
								}
							}
							wchar_t wy = (wchar_t)_intr::stringToUnsignedNumber<natural>(buff,16);
							WideToUtf8Filter flt;
							flt.input(wy);
							while (flt.hasItems()) strBuff.add(flt.output());
							x = iter.getNext();
							continue;
						}
					default: throw ErrorMessageException(THISLOCATION, "Neocekavana escape sekvence");
					//TODO: vymyslet lepsi vyjimku
				}
				strBuff.add(y);
			}
			x = iter.getNext();
		}
}


template<typename T>
PNode Parser<T>::parseObject(INode *container) {

	AutoArray<char, SmallAlloc<256> > name;
	Value obj = container;

	char c=iter.getNext();
	if (isspace(c)) c = iter.getNext();
	///empty object can be there
	if (c == '}') return obj;


	bool cont;
	do {

		while (isspace(c)) c= iter.getNext();
		if (c != '"') throw ParseError_t(THISLOCATION,ConstStrA());
		parseRawString();
		name.append(strBuff);

		char dcol = iter.getNext();
		while (isspace(dcol)) dcol = iter.getNext();
		if (dcol != ':')
				throw ParseError_t(THISLOCATION, ConstStrA(name));
		try {
			Value v = parse();
			obj->add(ConstStrA(name),v);
		} catch (Exception &e) {
			throw ParseError_t(THISLOCATION,name) << e;
		}
		name.clear();
		bool rep;
		do {
			c = iter.getNext();
			switch (c) {
			case ',': rep = false; cont = true;c = iter.getNext();break;
			case '}': rep = false; cont = false;break;
			default: if (isspace(c)) rep = true;
					else throw ParseError_t(THISLOCATION,name);
			}
		}while (rep);
	} while (cont);

	return obj;
}

template<typename T>
PNode Parser<T>::parseArray(INode *container) {

	Value arr = container;

	natural i = 0;

	char c=iter.peek();
	if (isspace(c)) {iter.skip();c = iter.peek();}
	///empty array can be there
	if (c == ']') {
		iter.skip();
		return arr;
	}


	bool cont;
	do {
		try {<commit_msg>fixed bug json parser multiple white characters in empty array is not error<commit_after>#include "jsondefs.h"
#include "jsonparser.h"


namespace LightSpeed {

double strtod(const char *string, char **endPtr);

namespace JSON {



template<typename T>
Value Parser<T>::parse() {

		while (true) {
			char x = iter.getNext();
			switch (x) {
				case '"': return parseString();
				case '{': return parseObject(factory->object());
				case '[': return parseArray(factory->array());
				case 't': parseCheck(strTrue+1);
						  if (sharedTrue == null) sharedTrue = factory->newValue(true);
						  return sharedTrue;
				case 'f': parseCheck(strFalse+1);
						  if (sharedFalse == null) sharedFalse = factory->newValue(false);
						  return sharedFalse;
				case 'n': parseCheck(strNull+1);
						  if (sharedNull == null) sharedNull = factory->newNullNode();
						  return sharedNull;
				case ' ':
				case '\t':
				case '\r':
				case '\n':
				case '\b':
				case '\a':
				case '\0': break;
				default: return parseValue(x);
			}
		}
	}


template<typename T>
PNode Parser<T>::parseString() {
	parseRawString();
	if (strBuff.empty()) {
		if (sharedEmptyStr == null) sharedEmptyStr = factory->newValue(ConstStrA());
		return sharedEmptyStr;
	} else {
		return factory->newValue(ConstStrA(strBuff));
	}
}


template<typename T>
void Parser<T>::parseRawString() {
	char x = iter.getNext();
		strBuff.clear();
		while (x != '"') {
			if (x != '\\') {
				strBuff.add(x);
			} else {
				x = iter.getNext();
				char y;

				switch (x) {
					case '"': y = x;break;
					case '\\':y = x;break;
					case '/': y = x ;break;
					case 'b': y  = '\b';break;
					case 'f': y = '\f';break;
					case 'n': y = '\n'; break;
					case 'r': y = '\r';break;
					case 't': y = '\t';break;
					case 'u': {
							char buff[5];
							buff[4] = 0;
							for (int i = 0; i < 4; i++) {
								char z  = iter.getNext();
								if ((z >= '0' && z <= '9')||(z >='A' && z <= 'F') || (z >= 'a' && z <= 'f')) {
									buff[i] = (char)z;
								} else {
									throw ParseError_t(THISLOCATION,strBuff);
								}
							}
							wchar_t wy = (wchar_t)_intr::stringToUnsignedNumber<natural>(buff,16);
							WideToUtf8Filter flt;
							flt.input(wy);
							while (flt.hasItems()) strBuff.add(flt.output());
							x = iter.getNext();
							continue;
						}
					default: throw ErrorMessageException(THISLOCATION, "Neocekavana escape sekvence");
					//TODO: vymyslet lepsi vyjimku
				}
				strBuff.add(y);
			}
			x = iter.getNext();
		}
}


template<typename T>
PNode Parser<T>::parseObject(INode *container) {

	AutoArray<char, SmallAlloc<256> > name;
	Value obj = container;

	char c=iter.getNext();
	while (isspace(c)) c = iter.getNext();
	///empty object can be there
	if (c == '}') return obj;


	bool cont;
	do {

		while (isspace(c)) c= iter.getNext();
		if (c != '"') throw ParseError_t(THISLOCATION,ConstStrA());
		parseRawString();
		name.append(strBuff);

		char dcol = iter.getNext();
		while (isspace(dcol)) dcol = iter.getNext();
		if (dcol != ':')
				throw ParseError_t(THISLOCATION, ConstStrA(name));
		try {
			Value v = parse();
			obj->add(ConstStrA(name),v);
		} catch (Exception &e) {
			throw ParseError_t(THISLOCATION,name) << e;
		}
		name.clear();
		bool rep;
		do {
			c = iter.getNext();
			switch (c) {
			case ',': rep = false; cont = true;c = iter.getNext();break;
			case '}': rep = false; cont = false;break;
			default: if (isspace(c)) rep = true;
					else throw ParseError_t(THISLOCATION,name);
			}
		}while (rep);
	} while (cont);

	return obj;
}

template<typename T>
PNode Parser<T>::parseArray(INode *container) {

	Value arr = container;

	natural i = 0;

	char c=iter.peek();
	while (isspace(c)) {iter.skip();c = iter.peek();}
	///empty array can be there
	if (c == ']') {
		iter.skip();
		return arr;
	}


	bool cont;
	do {
		try {
{'fraction_non_alphanumeric': 0.18129721281583747, 'fraction_numerical': 0.004688721021099245, 'mean_word_length': 2.868513853904282, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6802674', 'n_tokens_mistral': 3463, 'n_tokens_neox': 2768, 'n_words': 678}