<commit_before>/*
	AtlasProtocolDecoder.cpp
	----------------
	begin   : 1999.11.29
	copyright   : (C) 1999 by <NAME> (ZW)
	email   : <EMAIL>
*/

#include "AtlasObject.h"
#include "AtlasProtocol.h"
#include "AtlasPackedAsciiDecoder.h"

#include <memory.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>

void APackedAsciiDecoder::newStream()
{
	// reset decoder
	state = 1;
	nestd = 0;
	token = 0;
}

void APackedAsciiDecoder::feedStream(string& data)
{
	buffer.append(data);
}

int APackedAsciiDecoder::getToken()
{
	int tmp = token;
	token = 0;
	return tmp;
}

int APackedAsciiDecoder::hasTokens()
{
	int	pos;
   //int	pos1;
	//int	ndx;
	int	chk;
	string	typ;

	//printf("BEG TOKEN=%i\n", token);

	if (token == -1) return-1; // buffer overflow !!!

	do {
		chk = 0;

		//printf("BEG STATE=%i\n", state);
		//printf("BEG BUFFR=%s\n", buffer.c_str());

		// this is where we are gonna put the state machine
		switch (state) {
		case 1:// start of message
			if((pos=buffer.find('{')) == -1) break;
			buffer = buffer.substr(pos+1);
			// change states
			token = AProtocol::atlasMSGBEG;
			state = 2;
			chk = 1;
			break;

		case 2:
			pos = buffer.find_first_of("[(%$#])}");
			if (pos == -1) break;
			if (pos != 0)
			{
				// bad protcol character
				token = AProtocol::atlasERRTOK;
				state = 1;
				break;
			}
			typ = buffer.substr(0,1);
			if (typ == ")") {
				// end of list
				buffer = buffer.substr(1);
				token = AProtocol::atlasATREND;
				type = AProtocol::atlasLST;
				state = 2;
				break;
			}
			if (typ == ">") {
				// end of typed list
				buffer = buffer.substr(1);
				token = AProtocol::atlasATREND;
				type = AProtocol::atlasLST;
				state = 2;
				break;
			}
			if (typ == "]") {
				// end of map
				buffer = buffer.substr(1);
				token = AProtocol::atlasATREND;
				type = AProtocol::atlasMAP;
				state = 2;
				break;
			}
			if (typ == "}") {
				// end of message
				buffer = buffer.substr(1);
				token = AProtocol::atlasMSGEND;
				state = 1;
				break;
			}
			// must be an attribute == wait until we got the name
			if ((pos=buffer.find('=')) == -1) break;
			// get the name out before proc
			name = buffer.substr(1,pos-1);
			buffer = buffer.substr(pos+1);
			// got an attribute name
			if (typ == "<") {
				typ = buffer.substr(0,1);
				if (typ == "!") type = AProtocol::atlasLSTURI;
				if (typ == "@") type = AProtocol::atlasLSTINT;
				if (typ == "#") type = AProtocol::atlasLSTFLT;
				if (typ == "$") type = AProtocol::atlasLSTSTR;
				if (typ == "%") type = AProtocol::atlasLSTLNG;
			}
			if (typ == "(") type = AProtocol::atlasLST;
			if (typ == "[") type = AProtocol::atlasMAP;
			if (typ == "@") type = AProtocol::atlasINT;
			if (typ == "%") type = AProtocol::atlasLNG;
			if (typ == "#") type = AProtocol::atlasFLT;
			if (typ == "$") type = AProtocol::atlasSTR;
			// change states, wait for value
			token = AProtocol::atlasATRBEG;
			if (type==AProtocol::atlasLST || type==AProtocol::atlasMAP) {
				state = 2;
			} else {
				state = 3;
			}
			chk = 1;
			break;

		case 3:
			pos = buffer.find_first_of("[(%$#])}");
			if (pos == -1) break;
			// got an end marker, pull the data
			sval = buffer.substr(0,pos);<commit_msg>Stream parser changes for typed lists and uris completed<commit_after>/*
	AtlasProtocolDecoder.cpp
	----------------
	begin   : 1999.11.29
	copyright   : (C) 1999 by <NAME> (ZW)
	email   : <EMAIL>
*/

#include "AtlasObject.h"
#include "AtlasProtocol.h"
#include "AtlasPackedAsciiDecoder.h"

#include <memory.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>

void APackedAsciiDecoder::newStream()
{
	// reset decoder
	state = 1;
	nestd = 0;
	token = 0;
}

void APackedAsciiDecoder::feedStream(string& data)
{
	buffer.append(data);
}

int APackedAsciiDecoder::getToken()
{
	int tmp = token;
	token = 0;
	return tmp;
}

int APackedAsciiDecoder::hasTokens()
{
	int	pos;
   //int	pos1;
	//int	ndx;
	int	chk;
	string	typ;

	//printf("BEG TOKEN=%i\n", token);

	if (token == -1) return-1; // buffer overflow !!!

	do {
		chk = 0;

		//printf("BEG STATE=%i\n", state);
		//printf("BEG BUFFR=%s\n", buffer.c_str());

		// this is where we are gonna put the state machine
		switch (state) {
		case 1:// start of message
			if((pos=buffer.find('{')) == -1) break;
			buffer = buffer.substr(pos+1);
			// change states
			token = AProtocol::atlasMSGBEG;
			state = 2;
			chk = 1;
			break;

		case 2:
			pos = buffer.find_first_of("[<(%$#])>}");
			if (pos == -1) break;
			if (pos != 0)
			{
				// bad protcol character
				token = AProtocol::atlasERRTOK;
				state = 1;
				break;
			}
			typ = buffer.substr(0,1);
			if (typ == ")") {
				// end of list
				buffer = buffer.substr(1);
				token = AProtocol::atlasATREND;
				type = AProtocol::atlasLST;
				state = 2;
				break;
			}
			if (typ == ">") {
				// end of typed list
				buffer = buffer.substr(1);
				token = AProtocol::atlasATREND;
				type = AProtocol::atlasLST;
				state = 2;
				break;
			}
			if (typ == "]") {
				// end of map
				buffer = buffer.substr(1);
				token = AProtocol::atlasATREND;
				type = AProtocol::atlasMAP;
				state = 2;
				break;
			}
			if (typ == "}") {
				// end of message
				buffer = buffer.substr(1);
				token = AProtocol::atlasMSGEND;
				state = 1;
				break;
			}
			// must be an attribute == wait until we got the name
			if ((pos=buffer.find('=')) == -1) break;
			// get the name out before proc
			name = buffer.substr(1,pos-1);
			buffer = buffer.substr(pos+1);
			// got an attribute name
			if (typ == "<") {
				typ = buffer.substr(0,1);
				if (typ == "!") type = AProtocol::atlasLSTURI;
				if (typ == "@") type = AProtocol::atlasLSTINT;
				if (typ == "#") type = AProtocol::atlasLSTFLT;
				if (typ == "$") type = AProtocol::atlasLSTSTR;
				if (typ == "%") type = AProtocol::atlasLSTLNG;
			}
			if (typ == "(") type = AProtocol::atlasLST;
			if (typ == "[") type = AProtocol::atlasMAP;
			if (typ == "@") type = AProtocol::atlasINT;
			if (typ == "%") type = AProtocol::atlasLNG;
			if (typ == "#") type = AProtocol::atlasFLT;
			if (typ == "$") type = AProtocol::atlasSTR;
			// change states, wait for value
			token = AProtocol::<PASSWORD>;
			if (type==AProtocol::atlasLST || type==AProtocol::atlasMAP) {
				state = 2;
			} else {
				state = 3;
			}
			chk = 1;
			break;

		case 3:
			pos = buffer.find_first_of("[(<%$#]>)}");
			if (pos == -1) break;
			// got an end marker, pull the data
			sval = buffer.substr(0,pos);
{'fraction_non_alphanumeric': 0.18215384615384617, 'fraction_numerical': 0.015692307692307693, 'mean_word_length': 2.7512983266012694, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2254869', 'n_tokens_mistral': 2969, 'n_tokens_neox': 2464, 'n_words': 607}