<commit_before>#include "stdafx.h"

#include "SupSpi/SupSpi.h"
using namespace supspi;

#define WIN32_LEAN_AND_MEAN
#include <windows.h>

#include "richedit.h"
#include <windowsx.h>

#include "Colorer.h"

#include "NULLC/nullc_debug.h"
#include "NULLC/StrAlgo.h"

#include <set>

class ColorCodeCallback
{
public:
	ColorCodeCallback()
	{
		colorer = NULL;
		style = COLOR_CODE;
	}
	ColorCodeCallback(Colorer* col, int nStyle)
	{
		colorer = col;
		style = nStyle;
	}

	void operator()(const char* start, const char* end)
	{
		colorer->ColorCode(style, start, end);
	}
	void operator()(int style, const char* start, const char* end)
	{
		colorer->ColorCode(style, start, end);
	}
private:
	Colorer *colorer;
	int style;
};

namespace ColorerGrammar
{
	// Temporary variables
	unsigned int	varSize, varTop;
	bool	currValConst;
	std::string	logStr;

	const char *codeStart;

	std::vector<std::string>	typeInfo;
	std::vector<unsigned int>	callArgCount;

	std::set<InplaceStr>	variables;

	void MarkVar(char const* s, char const* e)
	{
		(void)s; (void)e;
		//printf("%.*s\r\n", int(e-s), s);
	}

	// Callbacks
	ColorCodeCallback ColorRWord, ColorVar, ColorVarDef, ColorFunc, ColorText, ColorChar, ColorReal, ColorInt, ColorBold, ColorErr, ColorComment;
	ColorCodeCallback ColorCode;

	//Error log
	ostringstream logStream;

	std::string	lastError;

	std::string tempStr;
	void SetTempStr(char const* s, char const* e)
	{
		tempStr.assign(s, e);
	}

	class LogError
	{
	public:
		LogError(): err(NULL){ }
		LogError(const char* str): err(str){ }

		void operator() (char const* s, char const* e)
		{
			(void)e;	// C4100
			assert(err);
			logStream << err << "\r\n";

			const char *begin = s;
			while((begin >= codeStart) && (*begin != '\n') && (*begin != '\r'))
				begin--;
			if(begin < s)
				begin++;

			const char *end = s;
			while((*end != '\r') && (*end != '\n') && (*end != 0))
				end++;

			if((end-begin) < 2048)
			{
				char line[2048];
				for(int k = 0; k < (end-begin); k++)
				{
					if(begin[k] < 0x20)
						line[k] = ' ';
					else
						line[k] = begin[k];
				}
				line[end-begin] = 0;
				logStream << "  at \"" << line << '\"';
			}else{
				logStream << "  at \"" << std::string(begin,end) << '\"';
			}
			logStream << "\r\n      ";
			for(unsigned int i = 0; i < (unsigned int)(s-begin); i++)
				logStream << ' ';
			logStream << "^\r\n";
		}
	private:
		const char* err;
	};

	class TypeNameP: public BaseP
	{
	public:
		TypeNameP(Rule a): m_a(a.getPtr()){ }
		virtual ~TypeNameP(){ }

		virtual bool	Parse(char** str, SpaceRule space)
		{
			if(space)
				space(str);
			char* curr = *str;
			m_a->Parse(str, NULL);
			if(curr == *str)
				return false;
			std::string type(curr, *str);
			for(unsigned int i = 0; i < typeInfo.size(); i++)
				if(typeInfo[i] == type)
					return true;
			return false;
		}
	protected:
		Rule m_a;
	};
	Rule	typenameP(Rule a){ return Rule(new TypeNameP(a)); }
	Rule	strWP(char* str){ return (lexemeD[strP(str) >> (epsP - (alnumP | '_'))]); }

	void	ParseSpace(char** str)
	{
		char *old = *str;
		for(;;)
		{
			while((unsigned char)((*str)[0] - 1) < ' ')
				(*str)++;
			if((*str)[0] == '/'){
				if((*str)[1] == '/')
				{
					while((*str)[0] != '\n' && (*str)[0] != '\0')
						(*str)++;
					ColorComment(COLOR_COMMENT, old, *str);
				}else if((*str)[1] == '*'){
					(*str) += 2;
					while(!((*str)[0] == '*' && (*str)[1] == '/') && (*str)[0] != '\0')
						(*str)++;
					if((*str)[0])
						(*str) += 2;
					ColorComment(COLOR_COMMENT, old, *str);
				}else{
					break;
				}
			}else{
				break;
			}
		}
	}

	class Grammar
	{
	public:
		void	InitGrammar()
		{
			constExpr		=	epsP[AssignVar<bool>(currValConst, false)] >>
				!strP("const")[ColorRWord][AssignVar<bool>(currValConst, true)];
			symb		=	graphP - alnumP - chP(')');
			symb2		=	graphP - alphaP;
			varname		=	lexemeD[(alphaP | '_') >> *(alnumP | '_')];
			typeName	=	varname - strP("return");

			arrayDef	=
				(
					chP('[')[ColorText] >>
					(term4_9 | epsP) >>
					(chP(']')[ColorText] | epsP[LogError("ERROR: closing ']' not found in array definition")]) >>
					!arrayDef
				);
			typeExpr	=
				(
					(strWP("auto") | typenameP(typeName))[ColorRWord] |
					(
						strP("typeof")[ColorRWord] >>
						(chP('(')[ColorText] | epsP[LogError("ERROR: '(' not found after 'typeof'")]) >>
						(term5 | epsP[LogError("ERROR: expression not found in 'typeof' statement")]) >>
						(chP(')')[ColorText] | epsP[LogError("ERROR: ')' not found after 'typeof' statement")])
					)
				) >>
				*(
					(strWP("ref")[ColorRWord] >> !(chP('(')[ColorText] >> !typeExpr >> *(chP(',')[ColorText] >> typeExpr) >> chP(')')[ColorText])) |
					arrayDef
				);

			classdef	=
				((strP("align")[ColorRWord] >> '(' >> intP[ColorReal] >> ')') | (strP("noalign")[ColorRWord] | epsP)) >>
				strP("class")[ColorRWord] >>
				(varname[StartType][ColorRWord] | epsP[LogError("ERROR: class name expected")]) >>
				(chP('{') | epsP[LogError("ERROR: '{' not found after class name")])[ColorText][BlockBegin] >>
				*(
					typeDef |
					funcdef |
					(
						typeExpr >>
						(
							(
								(varname - typenameP(varname))[ColorVarDef] >>
								chP('{')[ColorText] >>
								(strP("get")[ColorRWord] | epsP[LogError("ERROR: 'get' is expected after '{'")]) >>
								(block | epsP[LogError("ERROR: function body expected after 'get'")]) >>
								!(
									strP("set")[ColorRWord] >>
									!(
										chP('(')[ColorText] >>
										((varname - typenameP(varname))[ColorVarDef] | epsP[LogError("ERROR: r-value name not found")]) >>
										(chP(')')[ColorText] | epsP[LogError("ERROR: ')' is expected after r-value name")])
									) >> 
									(block | epsP[LogError("ERROR: function body expected after 'set'")])
								) >>
								(chP('}')[ColorText] | epsP[LogError("ERROR: '}' is expected after property")])
							) | (
								((varname - typenameP(varname))[ColorVarDef][AddVar] | epsP[LogError("ERROR: variable name not found after type")]) >>
								*(
									chP(',')[ColorText] >>
									((varname - typenameP(varname))[ColorVarDef][AddVar] | epsP[LogError("ERROR: variable name not found after ','")])
								) 
							)
						)>>
						(chP(';') | epsP[LogError("ERROR: ';' expected after variable list")])[ColorText]
					)
				) >>
				(chP('}') | epsP[LogError("ERROR: '}' not found after class definition")])[ColorText][BlockEnd];

			funccall	=	(typeExpr | varname)[ColorFunc] >> fcallpart;

			fcallpart	=
				strP("(")[ColorBold][PushBackVal<std::vector<unsigned int>, unsigned int>(callArgCount, 0)] >>
				!(
					term5[ArrBackInc<std::vector<unsigned int> >(callArgCount)] >>
					*(
						strP(",")[ColorText] >> 
						(term5[ArrBackInc<std::vector<unsigned int> >(callArgCount)] | epsP[LogError("ERROR: unexpected symbol after ','")])
					)[OnError]
				) >>
				(strP(")")[ColorBold] | epsP[LogError("ERROR: ')' not found after function call")]);
			funcvars	=
				!(
					typeExpr >>
					constExpr >>
					((varname - typenameP(varname))[ColorVar][AddVar][ArrBackInc<std::vector<unsigned int> >(callArgCount)] | epsP[LogError("ERROR: variable name expected after type")]) >>
					!(chP('=')[ColorText] >> term4_9)
				) >>
				*(
					strP(",")[ColorText] >>
					(
						!(typeExpr >> constExpr) >>
						((varname - typenameP(varname))[ColorVar][AddVar][ArrBackInc<std::vector<unsigned int> >(callArgCount)] | epsP[LogError("ERROR: parameter name expected after ','")]) >>
						!(chP('=')[ColorText] >> term4_9)
					)
				)[OnError];
			funcdef		=
				!strWP("coroutine")[ColorRWord] >>
				(
					(strWP("auto")[ColorRWord] >> chP('(')[ColorBold]) |
					(typeExpr >>
						(
							(
								strP("operator")[ColorRWord] >>
								(
									strP("**") | strP("<=") | strP(">=") | strP("!=") | strP("==") | strP("<<") | strP(">>") | strP("&&") | strP("||") | strP("^^") | strP("()") |
									strP("+=") | strP("-=") | strP("*=") | strP("/=") | strP("**=") |
									chP('+') | chP('-') | chP('*') | chP('/') | chP('%') | chP('!') | chP('~') | chP('<') | chP('>') | chP('&') | chP('|') | chP('^') |
									(chP('[') >> chP(']')) | (chP('(') >> chP(')')) |
									chP('=')
								)[ColorText] >>
								chP('(')[ColorBold]
							) |
							(typenameP(typeName)[ColorRWord] >> (chP(':') | chP('.'))[ColorBold] >> (varname[ColorFunc] | epsP[LogError("ERROR: function name expected after ':'")]) >> chP('(')[ColorBold]) |
							(varname[ColorFunc] >> chP('(')[ColorBold])
						)[SetTempStr]
					)
				)[PushBackVal<std::vector<unsigned int>, unsigned int>(callArgCount, 0)][FuncAdd][BlockBegin] >>
				(
					(*(symb | digitP))[ColorErr] >>
					funcvars
				) >>
				(chP(')')[ColorBold][FuncEnd] | epsP[LogError("ERROR: ')' expected after function parameter list")]) >>
				(
					chP(';')[ColorBold] |
					(
						chP('{')[ColorBold] >>
						(code | epsP[LogError("ERROR: function body not found")]) >>
						(chP('}')[ColorBold][BlockEnd] | epsP[LogError("ERROR: '}' not found after function body")])
					) |
					epsP[LogError("ERROR: unexpected symbol after function header")]
				);

			postExpr	=	(chP('[')[ColorText] >> term5 >> chP(']')[ColorText]) |	(chP('.')[ColorText] >>	((varname[ColorFunc] >> fcallpart) | varname[ColorVar])) | fcallpart;
			appval		=	(varname - (strP("case") | strP("default")))[ColorVar][MarkVar] >> ~chP('(') >> *postExpr;
			addvarp		=
				(
				varname[ColorVarDef] >> epsP[AssignVar<unsigned int>(varSize,1)] >> 
				!chP('[')[LogError("ERROR: unexpected '[', array size must be specified after typename")]
				)[AddVar] >>
				((chP('=')[ColorText] >> (term5 | epsP[LogError("ERROR: expression not found after '='")])) | epsP);
			vardefsub	=	addvarp >> *(chP(',')[ColorText] >> vardefsub);
			vardef		=
				((strP("align")[ColorRWord] >> '(' >> intP[ColorReal] >> ')') | (strP("noalign")[ColorRWord] | epsP)) >>
				typeExpr >> 
				(
					(fcallpart >> *postExpr) |
					(
						constExpr >>
						(vardefsub | epsP[LogError("ERROR: variable definition after typename is incorrect")])
					)
				);

			ifExpr			=
				strWP("if")[ColorRWord] >>
				(
					('(' | epsP[LogError("ERROR: '(' not found after 'if'")])[ColorText] >>
					(term5 | epsP[LogError("ERROR: condition not found in 'if' statement")]) >>
					(')' | epsP[LogError("ERROR: ')' not found after 'if' condition")])[ColorText]
				) >>
				(expr | epsP[LogError("ERROR: expression not found after 'if' statement")]) >>
				(
					(strP("else")[ColorRWord] >> (expr | epsP[LogError("ERROR: expression not found after 'else'")])) |
					epsP
				);

			forExpr			=
				strWP("for")[ColorRWord] >>
				('(' | epsP[LogError("ERROR: '(' not found after 'for'")])[ColorText] >>
				(
					(!typeExpr >> varname[ColorVarDef] >> strWP("in")[ColorRWord] >> term5 >> *(chP(',')[ColorText] >> !typeExpr >> varname[ColorVarDef] >> strWP("in")[ColorRWord] >> term5)) |
					(
						(block | vardef | term5 | epsP ) >>
						(';' | epsP[LogError("ERROR: ';' not found after initializer in 'for'")])[ColorText] >>
						(term5 | epsP[LogError("ERROR: condition not found in 'for' statement")]) >>
						(';' | epsP[LogError("ERROR: ';' not found after condition in 'for'")])[ColorText] >>
						(block | term5 | epsP)
					)
				) >>
				(')' | epsP[LogError("ERROR: ')' not found after 'for' statement")])[ColorText] >>
				(expr | epsP[LogError("ERROR: expression not found after 'for' statement")]);

			whileExpr		=
				strWP("while")[ColorRWord] >>
				(
					('(' | epsP[LogError("ERROR: '(' not found after 'while'")])[ColorText] >>
					(term5 | epsP[LogError("ERROR: condition not found in 'while' statement")]) >>
					(')' | epsP[LogError("ERROR: ')' not found after 'while' condition")])[ColorText]
				) >>
				(expr | epsP[LogError("ERROR: expression not found after 'while' statement")]);

			dowhileExpr		=
				strWP("do")[ColorRWord] >>
				(expr | epsP[LogError("ERROR: expression or block not found after 'do'")]) >>
				(strP("while")[ColorRWord] | epsP[LogError("ERROR: 'while' not found after body of 'do'")]) >>
				('(' | epsP[LogError("ERROR: '(' not found after 'while'")])[ColorText] >>
				(term5 | epsP[LogError("ERROR: condition not found in 'while' statement")]) >>
				(')' | epsP[LogError("ERROR: ')' not found after 'while' condition")])[ColorText] >>
				(';' | epsP[LogError("ERROR: ';' expected after 'do...while' statement")])[ColorText];

			switchExpr		=
				strWP("switch")[ColorRWord] >>
				(
					('(' | epsP[LogError("ERROR: '(' not found after 'switch'")])[ColorText] >>
					(term5 | epsP[LogError("ERROR: condition not found in 'switch' statement")]) >>
					(')' | epsP[LogError("ERROR: ')' not found after 'switch' condition")])[ColorText]
				) >>
				('{' | epsP[LogError("ERROR: '{' expected")])[ColorBold] >> 
				*(
					strWP("case")[ColorRWord] >>
					(term5 | epsP[LogError("ERROR: case condition expected")]) >>
					(':' | epsP[LogError("ERROR: ':' not found after case condition")])[ColorText] >>
					*expr
				) >>
				!(
					strWP("default")[ColorRWord] >>
					(':' | epsP[LogError("ERROR: ':' not found after default")])[ColorText] >>
					*expr
				) >>
				('}' | epsP[LogError("ERROR: '}' expected")])[ColorBold];

			typeDef			=	strWP("typedef")[ColorRWord] >>
				(typeExpr | epsP[LogError("ERROR: typename expected after typedef")]) >>
				(
					(typeExpr >> epsP[LogError("ERROR: there is already a type or an alias with the same name")]) |
					varname[StartType][ColorRWord] |
					epsP[LogError("ERROR: alias name expected after typename in typedef expression")]
				) >>
				(chP(';') | epsP[LogError("ERROR: ';' expected after typedef")]);

			returnExpr		=	(strWP("return") | strWP("yield"))[ColorRWord] >> (term5 | epsP) >> (+(';' >> epsP)[ColorBold] | epsP[LogError("ERROR: return statement must be followed by ';'")]);
			breakExpr		=	strWP("break")[ColorRWord] >> (term4_9 | epsP) >> (+chP(';')[ColorBold] | epsP[LogError("ERROR: break statement must be followed by ';'")]);
			continueExpr	=	strWP("continue")[ColorRWord] >> (term4_9 | epsP) >> (+chP(';')[ColorBold] | epsP[LogError("ERROR: continue statement must be followed by ';'")]);

			group		=	chP('(')[ColorText] >> term5 >> chP(')')[ColorText];
			term1		=
				funcdef |
				strWP("nullptr")[ColorRWord] |
				(strWP("sizeof")[ColorRWord] >> chP('(')[ColorText] >> (typeExpr | term5) >> chP(')')[ColorText]) |
				(chP('&')[ColorText] >> appval) |
				((strP("--") | strP("++"))[ColorText] >> appval[GetVar]) | 
				(+chP('-')[ColorText] >> term1) | (+chP('+')[ColorText] >> term1) | ((chP('!') | '~')[ColorText] >> term1) |
				(!chP('@')[ColorText] >> chP('\"')[ColorText] >> *((strP("\\\"") | strP("\\r") | strP("\\n") | strP("\\\'") | strP("\\\\") | strP("\\t") | strP("\\0"))[ColorReal] | (anycharP[ColorChar] - chP('\"'))) >> chP('\"')[ColorText] >> *postExpr) |
				lexemeD[strP("0x") >> +(digitP | chP('a') | chP('b') | chP('c') | chP('d') | chP('e') | chP('f') | chP('A') | chP('B') | chP('C') | chP('D') | chP('E') | chP('F'))][ColorReal] |
				longestD[(intP >> (chP('l') | chP('b') | epsP)) | (realP >> (chP('f') | epsP))][ColorReal] |
				lexemeD[chP('\'')[ColorText] >> ((chP('\\') >> anycharP)[ColorReal] | anycharP[ColorChar]) >> chP('\'')[ColorText]] |
				(chP('{')[ColorText] >> term5 >> *(chP(',')[ColorText] >> term5) >> chP('}')[ColorText] >> *postExpr) |
				(strWP("new")[ColorRWord] >> typeExpr >> !((chP('[')[ColorText] >> term4_9 >> chP(']')[ColorText]) | fcallpart)) |
				(group >> *postExpr) |
				(funccall[FuncCall] >> *postExpr) |
				(typeExpr) |
				(((+chP('*')[ColorText] >> term1) | appval) >> (strP("++")[ColorText] | strP("--")[ColorText] | (chP('.')[ColorText] >> funccall >> *postExpr) | epsP));
			term2	=	term1 >> *(strP("**")[ColorText] >> (term1 | epsP[LogError("ERROR: expression not found after operator **")]));
			term3	=	term2 >> *((chP('*') | chP('/') | chP('%'))[ColorText] >> (term2 | epsP[LogError("ERROR: expression not found after operator")]));
			term4	=	term3 >> *((chP('+') | chP('-'))[ColorText] >> (term3 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_1	=	term4 >> *((strP("<<") | strP(">>"))[ColorText] >> (term4 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_2	=	term4_1 >> *((strP("<=") | strP(">=") | chP('<') | chP('>'))[ColorText] >> (term4_1 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_4	=	term4_2 >> *((strP("==") | strP("!="))[ColorText] >> (term4_2 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_6	=	term4_4 >> *((strP("&&") | strP("||") | strP("^^") | chP('&') | chP('|') | chP('^'))[ColorText] >> (term4_4 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_9	=	term4_6 >> 
				!(
					chP('?')[ColorText] >>
					(term5 | epsP[LogError("ERROR: expression not found after operator ?")]) >>
					(chP(':')[ColorText] | epsP[LogError("ERROR: ':' not found in conditional statement")]) >>
					(term5 | epsP[LogError("ERROR: expression not found after ':' in conditional statement")])
				);
			term5	=	(
				term4_9 >>
				!(
					(chP('=')[ColorText] >> term5) |<commit_msg>SuperCalc: Colorer supports list comprehension and correctly parses modify-and-set operators.<commit_after>#include "stdafx.h"

#include "SupSpi/SupSpi.h"
using namespace supspi;

#define WIN32_LEAN_AND_MEAN
#include <windows.h>

#include "richedit.h"
#include <windowsx.h>

#include "Colorer.h"

#include "NULLC/nullc_debug.h"
#include "NULLC/StrAlgo.h"

#include <set>

class ColorCodeCallback
{
public:
	ColorCodeCallback()
	{
		colorer = NULL;
		style = COLOR_CODE;
	}
	ColorCodeCallback(Colorer* col, int nStyle)
	{
		colorer = col;
		style = nStyle;
	}

	void operator()(const char* start, const char* end)
	{
		colorer->ColorCode(style, start, end);
	}
	void operator()(int style, const char* start, const char* end)
	{
		colorer->ColorCode(style, start, end);
	}
private:
	Colorer *colorer;
	int style;
};

namespace ColorerGrammar
{
	// Temporary variables
	unsigned int	varSize, varTop;
	bool	currValConst;
	std::string	logStr;

	const char *codeStart;

	std::vector<std::string>	typeInfo;
	std::vector<unsigned int>	callArgCount;

	std::set<InplaceStr>	variables;

	void MarkVar(char const* s, char const* e)
	{
		(void)s; (void)e;
		//printf("%.*s\r\n", int(e-s), s);
	}

	// Callbacks
	ColorCodeCallback ColorRWord, ColorVar, ColorVarDef, ColorFunc, ColorText, ColorChar, ColorReal, ColorInt, ColorBold, ColorErr, ColorComment;
	ColorCodeCallback ColorCode;

	//Error log
	ostringstream logStream;

	std::string	lastError;

	std::string tempStr;
	void SetTempStr(char const* s, char const* e)
	{
		tempStr.assign(s, e);
	}

	class LogError
	{
	public:
		LogError(): err(NULL){ }
		LogError(const char* str): err(str){ }

		void operator() (char const* s, char const* e)
		{
			(void)e;	// C4100
			assert(err);
			logStream << err << "\r\n";

			const char *begin = s;
			while((begin >= codeStart) && (*begin != '\n') && (*begin != '\r'))
				begin--;
			if(begin < s)
				begin++;

			const char *end = s;
			while((*end != '\r') && (*end != '\n') && (*end != 0))
				end++;

			if((end-begin) < 2048)
			{
				char line[2048];
				for(int k = 0; k < (end-begin); k++)
				{
					if(begin[k] < 0x20)
						line[k] = ' ';
					else
						line[k] = begin[k];
				}
				line[end-begin] = 0;
				logStream << "  at \"" << line << '\"';
			}else{
				logStream << "  at \"" << std::string(begin,end) << '\"';
			}
			logStream << "\r\n      ";
			for(unsigned int i = 0; i < (unsigned int)(s-begin); i++)
				logStream << ' ';
			logStream << "^\r\n";
		}
	private:
		const char* err;
	};

	class TypeNameP: public BaseP
	{
	public:
		TypeNameP(Rule a): m_a(a.getPtr()){ }
		virtual ~TypeNameP(){ }

		virtual bool	Parse(char** str, SpaceRule space)
		{
			if(space)
				space(str);
			char* curr = *str;
			m_a->Parse(str, NULL);
			if(curr == *str)
				return false;
			std::string type(curr, *str);
			for(unsigned int i = 0; i < typeInfo.size(); i++)
				if(typeInfo[i] == type)
					return true;
			return false;
		}
	protected:
		Rule m_a;
	};
	Rule	typenameP(Rule a){ return Rule(new TypeNameP(a)); }
	Rule	strWP(char* str){ return (lexemeD[strP(str) >> (epsP - (alnumP | '_'))]); }

	void	ParseSpace(char** str)
	{
		char *old = *str;
		for(;;)
		{
			while((unsigned char)((*str)[0] - 1) < ' ')
				(*str)++;
			if((*str)[0] == '/'){
				if((*str)[1] == '/')
				{
					while((*str)[0] != '\n' && (*str)[0] != '\0')
						(*str)++;
					ColorComment(COLOR_COMMENT, old, *str);
				}else if((*str)[1] == '*'){
					(*str) += 2;
					while(!((*str)[0] == '*' && (*str)[1] == '/') && (*str)[0] != '\0')
						(*str)++;
					if((*str)[0])
						(*str) += 2;
					ColorComment(COLOR_COMMENT, old, *str);
				}else{
					break;
				}
			}else{
				break;
			}
		}
	}

	class Grammar
	{
	public:
		void	InitGrammar()
		{
			constExpr		=	epsP[AssignVar<bool>(currValConst, false)] >>
				!strP("const")[ColorRWord][AssignVar<bool>(currValConst, true)];
			symb		=	graphP - alnumP - chP(')');
			symb2		=	graphP - alphaP;
			varname		=	lexemeD[(alphaP | '_') >> *(alnumP | '_')];
			typeName	=	varname - strP("return");

			arrayDef	=
				(
					chP('[')[ColorText] >>
					(term4_9 | epsP) >>
					(chP(']')[ColorText] | epsP[LogError("ERROR: closing ']' not found in array definition")]) >>
					!arrayDef
				);
			typeExpr	=
				(
					(strWP("auto") | typenameP(typeName))[ColorRWord] |
					(
						strP("typeof")[ColorRWord] >>
						(chP('(')[ColorText] | epsP[LogError("ERROR: '(' not found after 'typeof'")]) >>
						(term5 | epsP[LogError("ERROR: expression not found in 'typeof' statement")]) >>
						(chP(')')[ColorText] | epsP[LogError("ERROR: ')' not found after 'typeof' statement")])
					)
				) >>
				*(
					(strWP("ref")[ColorRWord] >> !(chP('(')[ColorText] >> !typeExpr >> *(chP(',')[ColorText] >> typeExpr) >> chP(')')[ColorText])) |
					arrayDef
				);

			classdef	=
				((strP("align")[ColorRWord] >> '(' >> intP[ColorReal] >> ')') | (strP("noalign")[ColorRWord] | epsP)) >>
				strP("class")[ColorRWord] >>
				(varname[StartType][ColorRWord] | epsP[LogError("ERROR: class name expected")]) >>
				(chP('{') | epsP[LogError("ERROR: '{' not found after class name")])[ColorText][BlockBegin] >>
				*(
					typeDef |
					funcdef |
					(
						typeExpr >>
						(
							(
								(varname - typenameP(varname))[ColorVarDef] >>
								chP('{')[ColorText] >>
								(strP("get")[ColorRWord] | epsP[LogError("ERROR: 'get' is expected after '{'")]) >>
								(block | epsP[LogError("ERROR: function body expected after 'get'")]) >>
								!(
									strP("set")[ColorRWord] >>
									!(
										chP('(')[ColorText] >>
										((varname - typenameP(varname))[ColorVarDef] | epsP[LogError("ERROR: r-value name not found")]) >>
										(chP(')')[ColorText] | epsP[LogError("ERROR: ')' is expected after r-value name")])
									) >> 
									(block | epsP[LogError("ERROR: function body expected after 'set'")])
								) >>
								(chP('}')[ColorText] | epsP[LogError("ERROR: '}' is expected after property")])
							) | (
								((varname - typenameP(varname))[ColorVarDef][AddVar] | epsP[LogError("ERROR: variable name not found after type")]) >>
								*(
									chP(',')[ColorText] >>
									((varname - typenameP(varname))[ColorVarDef][AddVar] | epsP[LogError("ERROR: variable name not found after ','")])
								) 
							)
						)>>
						(chP(';') | epsP[LogError("ERROR: ';' expected after variable list")])[ColorText]
					)
				) >>
				(chP('}') | epsP[LogError("ERROR: '}' not found after class definition")])[ColorText][BlockEnd];

			funccall	=	(typeExpr | varname)[ColorFunc] >> fcallpart;

			fcallpart	=
				strP("(")[ColorBold][PushBackVal<std::vector<unsigned int>, unsigned int>(callArgCount, 0)] >>
				!(
					term5[ArrBackInc<std::vector<unsigned int> >(callArgCount)] >>
					*(
						strP(",")[ColorText] >> 
						(term5[ArrBackInc<std::vector<unsigned int> >(callArgCount)] | epsP[LogError("ERROR: unexpected symbol after ','")])
					)[OnError]
				) >>
				(strP(")")[ColorBold] | epsP[LogError("ERROR: ')' not found after function call")]);
			funcvars	=
				!(
					typeExpr >>
					constExpr >>
					((varname - typenameP(varname))[ColorVar][AddVar][ArrBackInc<std::vector<unsigned int> >(callArgCount)] | epsP[LogError("ERROR: variable name expected after type")]) >>
					!(chP('=')[ColorText] >> term4_9)
				) >>
				*(
					strP(",")[ColorText] >>
					(
						!(typeExpr >> constExpr) >>
						((varname - typenameP(varname))[ColorVar][AddVar][ArrBackInc<std::vector<unsigned int> >(callArgCount)] | epsP[LogError("ERROR: parameter name expected after ','")]) >>
						!(chP('=')[ColorText] >> term4_9)
					)
				)[OnError];
			funcdef		=
				!strWP("coroutine")[ColorRWord] >>
				(
					(strWP("auto")[ColorRWord] >> chP('(')[ColorBold]) |
					(typeExpr >>
						(
							(
								strP("operator")[ColorRWord] >>
								(
									strP("**") | strP("<=") | strP(">=") | strP("!=") | strP("==") | strP("<<") | strP(">>") | strP("&&") | strP("||") | strP("^^") | strP("()") |
									strP("+=") | strP("-=") | strP("*=") | strP("/=") | strP("**=") |
									chP('+') | chP('-') | chP('*') | chP('/') | chP('%') | chP('!') | chP('~') | chP('<') | chP('>') | chP('&') | chP('|') | chP('^') |
									(chP('[') >> chP(']')) | (chP('(') >> chP(')')) |
									chP('=')
								)[ColorText] >>
								chP('(')[ColorBold]
							) |
							(typenameP(typeName)[ColorRWord] >> (chP(':') | chP('.'))[ColorBold] >> (varname[ColorFunc] | epsP[LogError("ERROR: function name expected after ':'")]) >> chP('(')[ColorBold]) |
							(varname[ColorFunc] >> chP('(')[ColorBold])
						)[SetTempStr]
					)
				)[PushBackVal<std::vector<unsigned int>, unsigned int>(callArgCount, 0)][FuncAdd][BlockBegin] >>
				(
					(*(symb | digitP))[ColorErr] >>
					funcvars
				) >>
				(chP(')')[ColorBold][FuncEnd] | epsP[LogError("ERROR: ')' expected after function parameter list")]) >>
				(
					chP(';')[ColorBold] |
					(
						chP('{')[ColorBold] >>
						(code | epsP[LogError("ERROR: function body not found")]) >>
						(chP('}')[ColorBold][BlockEnd] | epsP[LogError("ERROR: '}' not found after function body")])
					) |
					epsP[LogError("ERROR: unexpected symbol after function header")]
				);

			postExpr	=	(chP('[')[ColorText] >> term5 >> chP(']')[ColorText]) |	(chP('.')[ColorText] >>	((varname[ColorFunc] >> fcallpart) | varname[ColorVar])) | fcallpart;
			appval		=	(varname - (strP("case") | strP("default")))[ColorVar][MarkVar] >> ~chP('(') >> *postExpr;
			addvarp		=
				(
				varname[ColorVarDef] >> epsP[AssignVar<unsigned int>(varSize,1)] >> 
				!chP('[')[LogError("ERROR: unexpected '[', array size must be specified after typename")]
				)[AddVar] >>
				((chP('=')[ColorText] >> (term5 | epsP[LogError("ERROR: expression not found after '='")])) | epsP);
			vardefsub	=	addvarp >> *(chP(',')[ColorText] >> vardefsub);
			vardef		=
				((strP("align")[ColorRWord] >> '(' >> intP[ColorReal] >> ')') | (strP("noalign")[ColorRWord] | epsP)) >>
				typeExpr >> 
				(
					(fcallpart >> *postExpr) |
					(
						constExpr >>
						(vardefsub | epsP[LogError("ERROR: variable definition after typename is incorrect")])
					)
				);

			ifExpr			=
				strWP("if")[ColorRWord] >>
				(
					('(' | epsP[LogError("ERROR: '(' not found after 'if'")])[ColorText] >>
					(term5 | epsP[LogError("ERROR: condition not found in 'if' statement")]) >>
					(')' | epsP[LogError("ERROR: ')' not found after 'if' condition")])[ColorText]
				) >>
				(expr | epsP[LogError("ERROR: expression not found after 'if' statement")]) >>
				(
					(strP("else")[ColorRWord] >> (expr | epsP[LogError("ERROR: expression not found after 'else'")])) |
					epsP
				);

			forExpr			=
				strWP("for")[ColorRWord] >>
				('(' | epsP[LogError("ERROR: '(' not found after 'for'")])[ColorText] >>
				(
					(!typeExpr >> varname[ColorVarDef] >> strWP("in")[ColorRWord] >> term5 >> *(chP(',')[ColorText] >> !typeExpr >> varname[ColorVarDef] >> strWP("in")[ColorRWord] >> term5)) |
					(
						(block | vardef | term5 | epsP ) >>
						(';' | epsP[LogError("ERROR: ';' not found after initializer in 'for'")])[ColorText] >>
						(term5 | epsP[LogError("ERROR: condition not found in 'for' statement")]) >>
						(';' | epsP[LogError("ERROR: ';' not found after condition in 'for'")])[ColorText] >>
						(block | term5 | epsP)
					)
				) >>
				(')' | epsP[LogError("ERROR: ')' not found after 'for' statement")])[ColorText] >>
				(expr | epsP[LogError("ERROR: expression not found after 'for' statement")]);

			whileExpr		=
				strWP("while")[ColorRWord] >>
				(
					('(' | epsP[LogError("ERROR: '(' not found after 'while'")])[ColorText] >>
					(term5 | epsP[LogError("ERROR: condition not found in 'while' statement")]) >>
					(')' | epsP[LogError("ERROR: ')' not found after 'while' condition")])[ColorText]
				) >>
				(expr | epsP[LogError("ERROR: expression not found after 'while' statement")]);

			dowhileExpr		=
				strWP("do")[ColorRWord] >>
				(expr | epsP[LogError("ERROR: expression or block not found after 'do'")]) >>
				(strP("while")[ColorRWord] | epsP[LogError("ERROR: 'while' not found after body of 'do'")]) >>
				('(' | epsP[LogError("ERROR: '(' not found after 'while'")])[ColorText] >>
				(term5 | epsP[LogError("ERROR: condition not found in 'while' statement")]) >>
				(')' | epsP[LogError("ERROR: ')' not found after 'while' condition")])[ColorText] >>
				(';' | epsP[LogError("ERROR: ';' expected after 'do...while' statement")])[ColorText];

			switchExpr		=
				strWP("switch")[ColorRWord] >>
				(
					('(' | epsP[LogError("ERROR: '(' not found after 'switch'")])[ColorText] >>
					(term5 | epsP[LogError("ERROR: condition not found in 'switch' statement")]) >>
					(')' | epsP[LogError("ERROR: ')' not found after 'switch' condition")])[ColorText]
				) >>
				('{' | epsP[LogError("ERROR: '{' expected")])[ColorBold] >> 
				*(
					strWP("case")[ColorRWord] >>
					(term5 | epsP[LogError("ERROR: case condition expected")]) >>
					(':' | epsP[LogError("ERROR: ':' not found after case condition")])[ColorText] >>
					*expr
				) >>
				!(
					strWP("default")[ColorRWord] >>
					(':' | epsP[LogError("ERROR: ':' not found after default")])[ColorText] >>
					*expr
				) >>
				('}' | epsP[LogError("ERROR: '}' expected")])[ColorBold];

			typeDef			=	strWP("typedef")[ColorRWord] >>
				(typeExpr | epsP[LogError("ERROR: typename expected after typedef")]) >>
				(
					(typeExpr >> epsP[LogError("ERROR: there is already a type or an alias with the same name")]) |
					varname[StartType][ColorRWord] |
					epsP[LogError("ERROR: alias name expected after typename in typedef expression")]
				) >>
				(chP(';') | epsP[LogError("ERROR: ';' expected after typedef")]);

			returnExpr		=	(strWP("return") | strWP("yield"))[ColorRWord] >> (term5 | epsP) >> (+(';' >> epsP)[ColorBold] | epsP[LogError("ERROR: return statement must be followed by ';'")]);
			breakExpr		=	strWP("break")[ColorRWord] >> (term4_9 | epsP) >> (+chP(';')[ColorBold] | epsP[LogError("ERROR: break statement must be followed by ';'")]);
			continueExpr	=	strWP("continue")[ColorRWord] >> (term4_9 | epsP) >> (+chP(';')[ColorBold] | epsP[LogError("ERROR: continue statement must be followed by ';'")]);

			group		=	chP('(')[ColorText] >> term5 >> chP(')')[ColorText];
			term1		=
				funcdef |
				strWP("nullptr")[ColorRWord] |
				(strWP("sizeof")[ColorRWord] >> chP('(')[ColorText] >> (typeExpr | term5) >> chP(')')[ColorText]) |
				(chP('&')[ColorText] >> appval) |
				((strP("--") | strP("++"))[ColorText] >> appval[GetVar]) | 
				(+chP('-')[ColorText] >> term1) | (+chP('+')[ColorText] >> term1) | ((chP('!') | '~')[ColorText] >> term1) |
				(!chP('@')[ColorText] >> chP('\"')[ColorText] >> *((strP("\\\"") | strP("\\r") | strP("\\n") | strP("\\\'") | strP("\\\\") | strP("\\t") | strP("\\0"))[ColorReal] | (anycharP[ColorChar] - chP('\"'))) >> chP('\"')[ColorText] >> *postExpr) |
				lexemeD[strP("0x") >> +(digitP | chP('a') | chP('b') | chP('c') | chP('d') | chP('e') | chP('f') | chP('A') | chP('B') | chP('C') | chP('D') | chP('E') | chP('F'))][ColorReal] |
				longestD[(intP >> (chP('l') | chP('b') | epsP)) | (realP >> (chP('f') | epsP))][ColorReal] |
				lexemeD[chP('\'')[ColorText] >> ((chP('\\') >> anycharP)[ColorReal] | anycharP[ColorChar]) >> chP('\'')[ColorText]] |
				(chP('{')[ColorText] >> ((forExpr >> code) | (term5 >> *(chP(',')[ColorText] >> term5))) >> chP('}')[ColorText] >> *postExpr) |
				(strWP("new")[ColorRWord] >> typeExpr >> !((chP('[')[ColorText] >> term4_9 >> chP(']')[ColorText]) | fcallpart)) |
				(group >> *postExpr) |
				(funccall[FuncCall] >> *postExpr) |
				(typeExpr) |
				(((+chP('*')[ColorText] >> term1) | appval) >> (strP("++")[ColorText] | strP("--")[ColorText] | (chP('.')[ColorText] >> funccall >> *postExpr) | epsP));
			term2	=	term1 >> *((strP("**") - strP("**="))[ColorText] >> (term1 | epsP[LogError("ERROR: expression not found after operator **")]));
			term3	=	term2 >> *(((chP('*') - strP("*=")) | (chP('/') - strP("/=")) | chP('%'))[ColorText] >> (term2 | epsP[LogError("ERROR: expression not found after operator")]));
			term4	=	term3 >> *(((chP('+') - strP("+=")) | (chP('-') - strP("-=")))[ColorText] >> (term3 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_1	=	term4 >> *((strP("<<") | strP(">>"))[ColorText] >> (term4 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_2	=	term4_1 >> *((strP("<=") | strP(">=") | chP('<') | chP('>'))[ColorText] >> (term4_1 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_4	=	term4_2 >> *((strP("==") | strP("!="))[ColorText] >> (term4_2 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_6	=	term4_4 >> *((strP("&&") | strP("||") | strP("^^") | chP('&') | chP('|') | chP('^'))[ColorText] >> (term4_4 | epsP[LogError("ERROR: expression not found after operator")]));
			term4_9	=	term4_6 >> 
				!(
					chP('?')[ColorText] >>
					(term5 | epsP[LogError("ERROR: expression not found after operator ?")]) >>
					(chP(':')[ColorText] | epsP[LogError("ERROR: ':' not found in conditional statement")]) >>
					(term5 | epsP[LogError("ERROR: expression not found after ':' in conditional statement")])
				);
			term5	=	(
				term4_9 >>
				!(
					(chP('=')[ColorText] >> term5) |
{'fraction_non_alphanumeric': 0.2358660320408637, 'fraction_numerical': 0.007023450197353146, 'mean_word_length': 3.655093218049176, 'pattern_counts': {'":': 0, '<': 119, '<?xml version=': 0, '>': 859, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4569829', 'n_tokens_mistral': 15540, 'n_tokens_neox': 12997, 'n_words': 2384}