<commit_before>/*******************************************************************************/
#include "WSLexer.h"


/*******************************************************************************/
WSLexer::WSLexer()
{
}


/*******************************************************************************/
WSLexer::~WSLexer()
{
}


/*******************************************************************************/
void WSLexer::Put_HttpRequest(const char* inHttpStr, const char* inHttpStrEnd)
{
	mpHttpStr		= inHttpStr;
	mpHttpStrEnd	= inHttpStrEnd  ? inHttpStrEnd 
									: inHttpStr + strlen(inHttpStr);
	mpCurrChar		= mpHttpStr;

	mLine			= 1;
}


/*******************************************************************************/
bool WSLexer::GetNextToken(Token* outToken)
{
	outToken->Clear();

	if( mpCurrChar == mpHttpStrEnd)
	{
		if(flagBracketsOpen || flagQuotesOpen)
		{
			throw exception("Error WSLexer: Don't closed brackets or quotes");
		}
		return false; // end of string
	}

	bool flagLongWord = false;

	outToken->ps = mpCurrChar;

	do
	{
		switch(*mpCurrChar)
		{
			case 32: // " "
			{
				if( !flagLongWord)
				{
					outToken->mType = wsSpaceType;
					outToken->pe = mpCurrChar;
					mpCurrChar = mpCurrChar + 1; // I only work with UTF-8 char, it has a length of 1.
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 42: // "*"
			case 43: // "+"
			case 46: // "."
			case 58: // ":"
			case 59: // ";"
			case 61: // "="
			{
				if( !flagLongWord)
				{
					outToken->mType = wsSymbolType;
					outToken->pe = mpCurrChar;
					mpCurrChar = mpCurrChar + 1;
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 34: // ""
			{
				if( !flagLongWord )
				{
					outToken->mType = wsQuotesSymbolType;
					outToken->pe = mpCurrChar;
					mpCurrChar = mpCurrChar + 1;

					flagQuotesOpen = flagQuotesOpen ? false : true;
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 28: // "("
			case 29: // ")"
			{
				if( !flagLongWord )
				{
					outToken->mType = wsBracketsSymbolType;
					outToken->pe = mpCurrChar;
					mpCurrChar = mpCurrChar + 1;

					flagBracketsOpen = flagBracketsOpen ? false : true;
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 10: // "\n"
			case 13: // "\r"
			{
				if( !flagLongWord )
				{
					outToken->mType = wsNewLineSymbolType;
					outToken->pe = mpCurrChar;
					mpCurrChar = mpCurrChar + 1;
					
					// if the next symbol is not \n or \r, increase Token::mLen by 1
					if( !(*(mpCurrChar) == 10 || *(mpCurrChar) == 13) )
					{
						outToken->mLine += 1;
					}
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			// All other symbol
			default:
			{
				outToken->mType = wsDefaultType;
				outToken->pe = mpCurrChar;
				mpCurrChar = mpCurrChar + 1;

				flagLongWord = true;
			}break;
		}
	}
	while(flagLongWord);

	outToken->mLen = (outToken->pe - outToken->ps) + 1;
	outToken->mPosition = (outToken->pe - mpHttpStr) + 1;

	return true;
}<commit_msg>fix WLexer pe + 1<commit_after>/*******************************************************************************/
#include "WSLexer.h"


/*******************************************************************************/
WSLexer::WSLexer()
{
}


/*******************************************************************************/
WSLexer::~WSLexer()
{
}


/*******************************************************************************/
void WSLexer::Put_HttpRequest(const char* inHttpStr, const char* inHttpStrEnd)
{
	mpHttpStr		= inHttpStr;
	mpHttpStrEnd	= inHttpStrEnd  ? inHttpStrEnd 
									: inHttpStr + strlen(inHttpStr);
	mpCurrChar		= mpHttpStr;

	mLine			= 1;
}


/*******************************************************************************/
bool WSLexer::GetNextToken(Token* outToken)
{
	outToken->Clear();

	if( mpCurrChar == mpHttpStrEnd )
	{
		if(flagBracketsOpen || flagQuotesOpen)
		{
			throw exception("Error WSLexer: Don't closed brackets or quotes");
		}
		return false; // end of string
	}

	bool flagLongWord = false;

	outToken->ps = mpCurrChar;

	do
	{
		switch(*mpCurrChar)
		{
			case 32: // " "
			{
				if( !flagLongWord)
				{
					mpCurrChar = mpCurrChar + 1; // I only work with UTF-8 char, it has a length of 1.

					outToken->mType = wsSpaceType;
					outToken->pe = mpCurrChar;
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 42: // "*"
			case 43: // "+"
			case 46: // "."
			case 58: // ":"
			case 59: // ";"
			case 61: // "="
			{
				if( !flagLongWord)
				{
					mpCurrChar = mpCurrChar + 1;

					outToken->mType = wsSymbolType;
					outToken->pe = mpCurrChar;
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 34: // ""
			{
				if( !flagLongWord )
				{
					mpCurrChar = mpCurrChar + 1;

					outToken->mType = wsQuotesSymbolType;
					outToken->pe = mpCurrChar;

					flagQuotesOpen = flagQuotesOpen ? false : true;
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 28: // "("
			case 29: // ")"
			{
				if( !flagLongWord )
				{
					mpCurrChar = mpCurrChar + 1;

					outToken->mType = wsBracketsSymbolType;
					outToken->pe = mpCurrChar;

					flagBracketsOpen = flagBracketsOpen ? false : true;
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			case 10: // "\n"
			case 13: // "\r"
			{
				if( !flagLongWord )
				{
					mpCurrChar = mpCurrChar + 1;

					outToken->mType = wsNewLineSymbolType;
					outToken->pe = mpCurrChar;
					
					// if the next symbol is not \n or \r, increase Token::mLen by 1
					if( !(*(mpCurrChar) == 10 || *(mpCurrChar) == 13) )
					{
						outToken->mLine += 1;
					}
				}
				else
				{
					flagLongWord = false;
				}
			}break;

			// All other symbol
			default:
			{
				mpCurrChar = mpCurrChar + 1;

				outToken->mType = wsDefaultType;
				outToken->pe = mpCurrChar;

				flagLongWord = true;
			}break;
		}
	}
	while(flagLongWord);

	outToken->mLen = (outToken->pe - outToken->ps) + 1;
	outToken->mPosition = (outToken->pe - mpHttpStr) + 1;

	return true;
}
{'fraction_non_alphanumeric': 0.25233951732063703, 'fraction_numerical': 0.013626662288622558, 'mean_word_length': 2.5877502944640756, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '853719', 'n_tokens_mistral': 2839, 'n_tokens_neox': 2081, 'n_words': 419}