<commit_before>/*

stacking xml parser

*/

'use strict';

const sInitial = 0;
const sDeclaration = 1;
const sPreElement = 2;
const sElement = 3;
const sAttribute = 5;
const sAttrNML = 6; // No Mans Land
const sValue = 7;
const sEndElement = 9;
const sContent = 11;
const sAttributeSpacer = 12;
const sComment = 13;
const sProcessingInstruction = 15;
const sEndDocument = 17;

String.prototype.replaceAll = function(search, replacement) {
    var target = this;
    return target.split(search).join(replacement);
};

function stateName(state) {
	if (state == sInitial) {
		return 'INITIAL';
	}
	else if (state == sDeclaration) {
		return 'DECLARATION';
	}
	else if (state == sElement) {
		return 'ELEMENT';
	}
	else if (state == sAttribute) {
		return 'ATTRIBUTE';
	}
	else if (state == sValue) {
		return 'VALUE';
	}
	else if (state == sEndElement) {
		return 'END_ELEMENT';
	}
	else if (state == sContent) {
		return 'CONTENT';
	}
	else if (state == sComment) {
		return 'COMMENT';
	}
	else if (state == sProcessingInstruction) {
		return 'PROCESSING_INSTRUCTION';
	}
	else if (state == sEndDocument) {
		return 'END_DOCUMENT';
	}
}

function reset(context) {
	context.state = sInitial;
	context.newState = sInitial;
	context.token = '';
	context.boundary = '<';
	context.lastElement = '';
	context.keepToken = false;
	context.position = 0;
}

// to create a push parser, pass in a callback function and omit the context parameter
// to create a pull parser, pass in null for the callback function and initially provide an empty object as the context
function staxParse(s,callback,context) {

	//comments - done trivially, needs hardening
	//processing instructions - done trivially, needs hardening
	//TODO CDATA segments

	if (context && context.newState) {
		if (!context.keepToken) context.token = '';
		context.state = context.newState;
	}
	else {
		context = {};
		reset(context);
	}

	var c;
	for (var i=context.position;i<s.length;i++) {
		c = s.charAt(i);

		if ((c == '\t') || (c == '\r') || (c == '\n')) { //other unicode spaces are not treated as whitespace
			c = ' ';
		}

		if (context.boundary.indexOf(c)>=0) {

			context.token = context.token.trim(); // nonstandard space handling
			context.keepToken = false;
			if (((context.state & 1) == 1) && (context.token != '')) {
				context.token = context.token.replaceAll('&amp;','&');
				context.token = context.token.replaceAll('&quot;','"');
				context.token = context.token.replaceAll('&apos;',"'");
				context.token = context.token.replaceAll('&gt;','>');
				context.token = context.token.replaceAll('&lt;','<');
				if (context.token.indexOf('&#') >= 0) {
					context.token = context.token.replace(/&(?:#([0-9]+)|#x([0-9a-fA-F]+));/g, function(match, group1, group2) {
						if (group2) {
							return String.fromCharCode(parseInt(group2,16));
						}
						else {
							return String.fromCharCode(group1);
						}
					});
				}

				if (callback) {
					callback(context.state,context.token);
				}
			}

			if (context.state == sInitial) {
				context.newState = sDeclaration;
				context.boundary = '>';
			}<commit_msg>Fixup comment and PI processing to match xmlWrite<commit_after>/*

stacking xml parser

*/

'use strict';

const sInitial = 0;
const sDeclaration = 1;
const sPreElement = 2;
const sElement = 3;
const sAttribute = 5;
const sAttrNML = 6; // No Mans Land
const sValue = 7;
const sEndElement = 9;
const sContent = 11;
const sAttributeSpacer = 12;
const sComment = 13;
const sProcessingInstruction = 15;
const sEndDocument = 17;

String.prototype.replaceAll = function(search, replacement) {
    var target = this;
    return target.split(search).join(replacement);
};

function stateName(state) {
	if (state == sInitial) {
		return 'INITIAL';
	}
	else if (state == sDeclaration) {
		return 'DECLARATION';
	}
	else if (state == sElement) {
		return 'ELEMENT';
	}
	else if (state == sAttribute) {
		return 'ATTRIBUTE';
	}
	else if (state == sValue) {
		return 'VALUE';
	}
	else if (state == sEndElement) {
		return 'END_ELEMENT';
	}
	else if (state == sContent) {
		return 'CONTENT';
	}
	else if (state == sComment) {
		return 'COMMENT';
	}
	else if (state == sProcessingInstruction) {
		return 'PROCESSING_INSTRUCTION';
	}
	else if (state == sEndDocument) {
		return 'END_DOCUMENT';
	}
}

function reset(context) {
	context.state = sInitial;
	context.newState = sInitial;
	context.token = '';
	context.boundary = '<';
	context.lastElement = '';
	context.keepToken = false;
	context.position = 0;
}

// to create a push parser, pass in a callback function and omit the context parameter
// to create a pull parser, pass in null for the callback function and initially provide an empty object as the context
function staxParse(s,callback,context) {

	//comments - done trivially, needs hardening
	//processing instructions - done trivially, needs hardening
	//TODO CDATA segments

	if (context && context.newState) {
		if (!context.keepToken) context.token = '';
		context.state = context.newState;
	}
	else {
		context = {};
		reset(context);
	}

	var c;
	for (var i=context.position;i<s.length;i++) {
		c = s.charAt(i);

		if ((c == '\t') || (c == '\r') || (c == '\n')) { //other unicode spaces are not treated as whitespace
			c = ' ';
		}

		if (context.boundary.indexOf(c)>=0) {

			context.token = context.token.trim(); // nonstandard space handling
			context.keepToken = false;
			if (((context.state & 1) == 1) && (context.token != '')) {
				context.token = context.token.replaceAll('&amp;','&');
				context.token = context.token.replaceAll('&quot;','"');
				context.token = context.token.replaceAll('&apos;',"'");
				context.token = context.token.replaceAll('&gt;','>');
				context.token = context.token.replaceAll('&lt;','<');
				if (context.token.indexOf('&#') >= 0) {
					context.token = context.token.replace(/&(?:#([0-9]+)|#x([0-9a-fA-F]+));/g, function(match, group1, group2) {
						if (group2) {
							return String.fromCharCode(parseInt(group2,16));
						}
						else {
							return String.fromCharCode(group1);
						}
					});
				}
				
				if (context.state == sComment) {
					context.token = context.token.replaceAll('-- ','');
					context.token = context.token.replaceAll(' --','').trim();
				}
				if (context.state == sProcessingInstruction) {
					context.token = context.token.replaceAll('?','').trim();
				}

				if (callback) {
					callback(context.state,context.token);
				}
			}

			if (context.state == sInitial) {
				context.newState = sDeclaration;
				context.boundary = '>';
			}
{'fraction_non_alphanumeric': 0.16156202143950996, 'fraction_numerical': 0.010413476263399694, 'mean_word_length': 3.4979338842975207, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6006990', 'n_tokens_mistral': 2479, 'n_tokens_neox': 2166, 'n_words': 548}