<commit_before>s9e = {};

/**
* @typedef {{
*   id:  !number,
*	pos: !number,
*	len: !number,
*	name: !string,
*	type: !number
* }}
*/
var Tag;

/**
* @typedef {{
*	name: !string,
*	pluginName: !string,
*	tagMate: !string,
*	context: !Object
* }}
*/
var StubTag;

s9e['TextFormatter'] = function()
{
	var
		/** @const */
		START_TAG        = 1,
		/** @const */
		END_TAG          = 2,
		/** @const */
		SELF_CLOSING_TAG = 3,

		/** @const */
		TRIM_CHARLIST = " \n\r\t\0\x0B",
		/** @const */
		rtrimRegExp = new RegExp('[' + TRIM_CHARLIST + ']*$'),
		/** @const */
		ltrimRegExp = new RegExp('^[' + TRIM_CHARLIST + ']*'),

		/** @type {!Object} */
		_log,

		/** @const @type {!Object} */
		tagsConfig = {/* DO NOT EDIT*/},
		/** @const @type {!Object} */
		filtersConfig = {/* DO NOT EDIT*/},
		/** @const @type {!Object} */
		pluginsConfig = {/* DO NOT EDIT*/},
		/** @const @type {!Object.<string, function(!string, !Object):Array>} */
		pluginParsers = {/* DO NOT EDIT*/},
		/** @const @type {!Object.<string, function>} */
		callbacks = {/* DO NOT EDIT*/},

		/** @type {!string} */
		text,
		/** @type {!Array.<Tag>} */
		unprocessedTags,
		/** @type {!Array.<Tag>} */
		processedTags,
		/** @type {!Object.<number,number>} */
		processedTagIds,
		/** @type {!Array.<StubTag>} */
		openTags,
		/** @type {!Object} */
		openStartTags,
		/** @type {!Object} */
		cntOpen,
		/** @type {!Object} */
		cntTotal,
		/** @type {!Tag} */
		currentTag,
		/** @type {!string} */
		currentAttribute,
		/** @type {!Object} */
		context,
		/** @type {!number} */
		pos,

		/** @const */
		xsl = '',

		/** @const */
		ENABLE_IE_WORKAROUNDS = 7,

		/** @const */
		ENABLE_LIVE_PREVIEW = true,

		MSXML = ENABLE_IE_WORKAROUNDS && !('XSLTProcessor' in window && 'DOMParser' in window);

	if (MSXML)
	{
		var xslt = new ActiveXObject('MSXML2.DOMDocument.3.0');
		xslt.async = false;
		xslt.validateOnParse = false;
		xslt.loadXML(xsl);
	}
	else
	{
		var xslt = new XSLTProcessor();
		xslt['importStylesheet'](new DOMParser().parseFromString(xsl, 'text/xml'));
	}

	if (ENABLE_IE_WORKAROUNDS && ENABLE_IE_WORKAROUNDS < 9)
	{
		if (!Array.prototype.forEach)
		{
			Array.prototype.forEach = function(fn)
			{
				var i = -1, cnt = this.length;

				while (++i < cnt)
				{
					fn(this[i], i);
				}
			}
		}

		if (!Array.prototype.some)
		{
			Array.prototype.some = function(fn)
			{
				var i = -1, cnt = this.length;

				while (++i < cnt)
				{
					if (fn(this[i], i))
					{
						return true;
					}
				}

				return false;
			}
		}
	}

	/**
	* @param {!Object}   obj
	* @param {!Function} callback
	*/
	function foreach(obj, callback)
	{
		for (var k in obj)
		{
			callback(obj[k], k);
		}
	}

	/**
	* @param {!RegExp} regexp
	* @param {!Array}  container
	*/
	function getMatches(regexp, container)
	{
		// reset the regexp
		regexp.lastIndex = 0;

		var cnt = 0,
			matches;

		while (matches = regexp.exec(text))
		{
			var pos   = matches.index,
				match = [[matches.shift(), pos]],
				str;

			while (str = matches.shift())
			{
				match.push([str, text.indexOf(str, pos)]);
				pos += str.length;
			}

			container.push(match);
			++cnt;

			if (regexp.lastIndex === 0)
			{
				regexp.lastIndex = 1;
			}
		}

		return cnt;
	}

	/**
	* @param {!Array} a1
	* @param {!Array} a2
	*/
	function contextAnd(a1, a2)
	{
		var ret = [];

		a1.forEach(function(v, k)
		{
			ret.push(v & a2[k]);
		});

		return ret;
	}

	/** @param {!string} _text */
	function reset(_text)
	{
		text = _text;

		_log = {
			'debug': [],
			'warning': [],
			'error': []
		};

		unprocessedTags = [];
		processedTags   = [];
		processedTagIds = {};
		openTags        = [];
		openStartTags   = {};
		cntOpen         = {};
		cntTotal        = {};

		delete currentTag;
		delete currentAttribute;
	}

	/**
	* @param {!string} type
	* @param {!Object} entry
	*/
	function log(type, entry)
	{
		if (currentTag)
		{
			entry['tagName'] = currentTag.name;
			entry['pluginName'] = currentTag.pluginName;

			if (currentAttribute)
			{
				entry['attrName'] = currentAttribute;
			}

			if (!('pos' in entry))
			{
				entry['pos'] = currentTag.pos;
				entry['len'] = currentTag.len;
			}
		}

		_log[type].push(entry);
	}

	function filter(attrVal, attrConf, filterConf)
	{
		switch (attrConf.type)
		{
			case 'url':
				var m =/^([a-z0-9]+):\/\/\S+(?:\/.*)?$/.exec(attrVal);

				if (!m)
				{
					return false;
				}

				if (!filterConf.allowedSchemes.test(m[1]))
				{
					log('error', {
						'msg'    : "URL scheme '%s' is not allowed",
						'params' : [m[1]]
					});
					return false;
				}

				var a = document.createElement('a');
				a.href = attrVal;

				if (filterConf.disallowedHosts
				 && filterConf.disallowedHosts.test(a.hostname))
				{
					log('error', {
						'msg'    : "URL host '%s' is not allowed",
						'params' : [a.hostname]
					});
					return false;
				}

				return attrVal.replace(/'/, '%27').replace(/"/, '%22');

			case 'identifier':
			case 'id':
				return /^[a-zA-Z0-9-_]+$/.test(attrVal) ? attrVal : false;

			case 'simpletext':
				return /^[a-zA-Z0-9\-+.,_ ]+$/.test(attrVal) ? attrVal : false;

			case 'text':
				return attrVal;

			case 'email':
				/**
				* NOTE: obviously, this is not meant to match precisely the whole set of theorically
				*       valid addresses. It's only there to catch honest mistakes. The actual
				*       validation should be performed by PHP's ext/filter.
				*/
				if (!/^[\w\.\-_]+@[\w\.\-_]+$/.test(attrVal))
				{
					return false;
				}

				if (attrConf.forceUrlencode)
				{
					return attrVal
						.split('')
						.map(function(c)
						{
							return '%' + c.charCodeAt(0).toString(16);
						})
						.join('');
				}

				return attrVal;

			case 'int':
			case 'integer':
				return /^-?[1-9][0-9]*$/.test(attrVal) ? attrVal : false;

			case 'float':
				return /^-?[0-9]+(?:\.[0-9]+)?(?:e[1-9][0-9]*)?$/i.test(attrVal) ? attrVal : false;

			case 'number':
				return /^[0-9]+$/.test(attrVal) ? attrVal : false;

			case 'uint':
				return /^(?:0|[1-9][0-9]*)$/.test(attrVal) ? attrVal : false;

			case 'range':
				if (!/^(?:0|-?[1-9][0-9]*)$/.test(attrVal))
				{
					return false;
				}

				if (attrVal < attrConf.min)
				{
					log('warning', {
						'msg'    : 'Value outside of range, adjusted up to %d',
						'params' : [attrConf.min]
					});
					return attrConf.min;
				}

				if (attrVal > attrConf.max)
				{
					log('warning', {
						'msg'    : 'Value outside of range, adjusted down to %d',
						'params' : [attrConf.max]
					});
					return attrConf.max;
				}

				return attrVal;

			case 'color':
				return /^(?:#[0-9a-f]{3,6}|[a-z]+)$/i.test(attrVal) ? attrVal : false;

			case 'regexp':
				var match = attrConf.regexp.exec(attrVal);

				if (!match)
				{
					return false;
				}

				if (attrConf.replaceWith)
				{
					/**
					* Two consecutive backslashes[1] are replaced with a single backslash.
					* A dollar sign followed by digits and preceded by a backslash[2] is preserved.
					* Otherwise, the corresponding match[3] is used.
					*/
					return attrConf.replaceWith.replace(
						/(\\\\)|(\\)?\$([0-9]+)/g,
						function (str, p1, p2, p3)
						{
							return (p1) ? '\\' : ((p2) ? '$' + p3 : match[p3]);
						}
					);
				}

				return attrVal;
		}

		log('debug', {
			'msg'    : "Unknown filter '%s'",
			'params' : [attrConf.type]
		});

		return false;
	}

	function output()
	{
		return asDOM();
	}

	function asDOM()
	{
		function createDOM(elName)
		{
			if (MSXML)
			{
				var DOM = new ActiveXObject('MSXML2.DOMDocument.3.0');
				DOM.async = false;
				DOM.validateOnParse = false;
				DOM.loadXML('<' + elName + '/>');

				return DOM;
			}

			return document.implementation.createDocument('', elName, null);
		}

		var stack = [],
			pos   = 0,
			i     = -1,
			cnt   = processedTags.length,
			DOM   = createDOM((cnt) ? 'rt' : 'pt'),
			el    = DOM.documentElement;

		function writeElement(tagName, content)
		{
			setTextContent(
				el.appendChild(DOM.createElement(tagName)),
				content
			);
		}

		function appendText(content)
		{
			el.appendChild(DOM.createTextNode(content));
		}

		function setTextContent(el, content)
		{
			if (ENABLE_IE_WORKAROUNDS && ENABLE_IE_WORKAROUNDS < 9 && !('textContent' in el))
			{
				el.appendChild(DOM.createTextNode(content));
			}
			else
			{
				el.nodeValue = content;
			}
		}

		while (++i < cnt)
		{
			var tag = processedTags[i];

			/**
			* Append the text that's between last tag and this one
			*/
			if (tag.pos > pos)
			{
				appendText(text.substr(pos, tag.pos - pos));
			}

			/**
			* Capture the part of the text that belongs to this tag then move the cursor past
			* current tag
			*/
			var tagText = text.substr(tag.pos, tag.len);
			pos = tag.pos + tag.len;

			var wsBefore = '',
				wsAfter  = '';

			if (tag.trimBefore)
			{
				wsBefore = tagText.substr(0, tag.trimBefore);
				tagText  = tagText.substr(tag.trimBefore);<commit_msg>TextFormatter: nodeValue doesn't work there<commit_after>s9e = {};

/**
* @typedef {{
*   id:  !number,
*	pos: !number,
*	len: !number,
*	name: !string,
*	type: !number
* }}
*/
var Tag;

/**
* @typedef {{
*	name: !string,
*	pluginName: !string,
*	tagMate: !string,
*	context: !Object
* }}
*/
var StubTag;

s9e['TextFormatter'] = function()
{
	var
		/** @const */
		START_TAG        = 1,
		/** @const */
		END_TAG          = 2,
		/** @const */
		SELF_CLOSING_TAG = 3,

		/** @const */
		TRIM_CHARLIST = " \n\r\t\0\x0B",
		/** @const */
		rtrimRegExp = new RegExp('[' + TRIM_CHARLIST + ']*$'),
		/** @const */
		ltrimRegExp = new RegExp('^[' + TRIM_CHARLIST + ']*'),

		/** @type {!Object} */
		_log,

		/** @const @type {!Object} */
		tagsConfig = {/* DO NOT EDIT*/},
		/** @const @type {!Object} */
		filtersConfig = {/* DO NOT EDIT*/},
		/** @const @type {!Object} */
		pluginsConfig = {/* DO NOT EDIT*/},
		/** @const @type {!Object.<string, function(!string, !Object):Array>} */
		pluginParsers = {/* DO NOT EDIT*/},
		/** @const @type {!Object.<string, function>} */
		callbacks = {/* DO NOT EDIT*/},

		/** @type {!string} */
		text,
		/** @type {!Array.<Tag>} */
		unprocessedTags,
		/** @type {!Array.<Tag>} */
		processedTags,
		/** @type {!Object.<number,number>} */
		processedTagIds,
		/** @type {!Array.<StubTag>} */
		openTags,
		/** @type {!Object} */
		openStartTags,
		/** @type {!Object} */
		cntOpen,
		/** @type {!Object} */
		cntTotal,
		/** @type {!Tag} */
		currentTag,
		/** @type {!string} */
		currentAttribute,
		/** @type {!Object} */
		context,
		/** @type {!number} */
		pos,

		/** @const */
		xsl = '',

		/** @const */
		ENABLE_IE_WORKAROUNDS = 7,

		/** @const */
		ENABLE_LIVE_PREVIEW = true,

		MSXML = ENABLE_IE_WORKAROUNDS && !('XSLTProcessor' in window && 'DOMParser' in window);

	if (MSXML)
	{
		var xslt = new ActiveXObject('MSXML2.DOMDocument.3.0');
		xslt.async = false;
		xslt.validateOnParse = false;
		xslt.loadXML(xsl);
	}
	else
	{
		var xslt = new XSLTProcessor();
		xslt['importStylesheet'](new DOMParser().parseFromString(xsl, 'text/xml'));
	}

	if (ENABLE_IE_WORKAROUNDS && ENABLE_IE_WORKAROUNDS < 9)
	{
		if (!Array.prototype.forEach)
		{
			Array.prototype.forEach = function(fn)
			{
				var i = -1, cnt = this.length;

				while (++i < cnt)
				{
					fn(this[i], i);
				}
			}
		}

		if (!Array.prototype.some)
		{
			Array.prototype.some = function(fn)
			{
				var i = -1, cnt = this.length;

				while (++i < cnt)
				{
					if (fn(this[i], i))
					{
						return true;
					}
				}

				return false;
			}
		}
	}

	/**
	* @param {!Object}   obj
	* @param {!Function} callback
	*/
	function foreach(obj, callback)
	{
		for (var k in obj)
		{
			callback(obj[k], k);
		}
	}

	/**
	* @param {!RegExp} regexp
	* @param {!Array}  container
	*/
	function getMatches(regexp, container)
	{
		// reset the regexp
		regexp.lastIndex = 0;

		var cnt = 0,
			matches;

		while (matches = regexp.exec(text))
		{
			var pos   = matches.index,
				match = [[matches.shift(), pos]],
				str;

			while (str = matches.shift())
			{
				match.push([str, text.indexOf(str, pos)]);
				pos += str.length;
			}

			container.push(match);
			++cnt;

			if (regexp.lastIndex === 0)
			{
				regexp.lastIndex = 1;
			}
		}

		return cnt;
	}

	/**
	* @param {!Array} a1
	* @param {!Array} a2
	*/
	function contextAnd(a1, a2)
	{
		var ret = [];

		a1.forEach(function(v, k)
		{
			ret.push(v & a2[k]);
		});

		return ret;
	}

	/** @param {!string} _text */
	function reset(_text)
	{
		text = _text;

		_log = {
			'debug': [],
			'warning': [],
			'error': []
		};

		unprocessedTags = [];
		processedTags   = [];
		processedTagIds = {};
		openTags        = [];
		openStartTags   = {};
		cntOpen         = {};
		cntTotal        = {};

		delete currentTag;
		delete currentAttribute;
	}

	/**
	* @param {!string} type
	* @param {!Object} entry
	*/
	function log(type, entry)
	{
		if (currentTag)
		{
			entry['tagName'] = currentTag.name;
			entry['pluginName'] = currentTag.pluginName;

			if (currentAttribute)
			{
				entry['attrName'] = currentAttribute;
			}

			if (!('pos' in entry))
			{
				entry['pos'] = currentTag.pos;
				entry['len'] = currentTag.len;
			}
		}

		_log[type].push(entry);
	}

	function filter(attrVal, attrConf, filterConf)
	{
		switch (attrConf.type)
		{
			case 'url':
				var m =/^([a-z0-9]+):\/\/\S+(?:\/.*)?$/.exec(attrVal);

				if (!m)
				{
					return false;
				}

				if (!filterConf.allowedSchemes.test(m[1]))
				{
					log('error', {
						'msg'    : "URL scheme '%s' is not allowed",
						'params' : [m[1]]
					});
					return false;
				}

				var a = document.createElement('a');
				a.href = attrVal;

				if (filterConf.disallowedHosts
				 && filterConf.disallowedHosts.test(a.hostname))
				{
					log('error', {
						'msg'    : "URL host '%s' is not allowed",
						'params' : [a.hostname]
					});
					return false;
				}

				return attrVal.replace(/'/, '%27').replace(/"/, '%22');

			case 'identifier':
			case 'id':
				return /^[a-zA-Z0-9-_]+$/.test(attrVal) ? attrVal : false;

			case 'simpletext':
				return /^[a-zA-Z0-9\-+.,_ ]+$/.test(attrVal) ? attrVal : false;

			case 'text':
				return attrVal;

			case 'email':
				/**
				* NOTE: obviously, this is not meant to match precisely the whole set of theorically
				*       valid addresses. It's only there to catch honest mistakes. The actual
				*       validation should be performed by PHP's ext/filter.
				*/
				if (!/^[\w\.\-_]+@[\w\.\-_]+$/.test(attrVal))
				{
					return false;
				}

				if (attrConf.forceUrlencode)
				{
					return attrVal
						.split('')
						.map(function(c)
						{
							return '%' + c.charCodeAt(0).toString(16);
						})
						.join('');
				}

				return attrVal;

			case 'int':
			case 'integer':
				return /^-?[1-9][0-9]*$/.test(attrVal) ? attrVal : false;

			case 'float':
				return /^-?[0-9]+(?:\.[0-9]+)?(?:e[1-9][0-9]*)?$/i.test(attrVal) ? attrVal : false;

			case 'number':
				return /^[0-9]+$/.test(attrVal) ? attrVal : false;

			case 'uint':
				return /^(?:0|[1-9][0-9]*)$/.test(attrVal) ? attrVal : false;

			case 'range':
				if (!/^(?:0|-?[1-9][0-9]*)$/.test(attrVal))
				{
					return false;
				}

				if (attrVal < attrConf.min)
				{
					log('warning', {
						'msg'    : 'Value outside of range, adjusted up to %d',
						'params' : [attrConf.min]
					});
					return attrConf.min;
				}

				if (attrVal > attrConf.max)
				{
					log('warning', {
						'msg'    : 'Value outside of range, adjusted down to %d',
						'params' : [attrConf.max]
					});
					return attrConf.max;
				}

				return attrVal;

			case 'color':
				return /^(?:#[0-9a-f]{3,6}|[a-z]+)$/i.test(attrVal) ? attrVal : false;

			case 'regexp':
				var match = attrConf.regexp.exec(attrVal);

				if (!match)
				{
					return false;
				}

				if (attrConf.replaceWith)
				{
					/**
					* Two consecutive backslashes[1] are replaced with a single backslash.
					* A dollar sign followed by digits and preceded by a backslash[2] is preserved.
					* Otherwise, the corresponding match[3] is used.
					*/
					return attrConf.replaceWith.replace(
						/(\\\\)|(\\)?\$([0-9]+)/g,
						function (str, p1, p2, p3)
						{
							return (p1) ? '\\' : ((p2) ? '$' + p3 : match[p3]);
						}
					);
				}

				return attrVal;
		}

		log('debug', {
			'msg'    : "Unknown filter '%s'",
			'params' : [attrConf.type]
		});

		return false;
	}

	function output()
	{
		return asDOM();
	}

	function asDOM()
	{
		function createDOM(elName)
		{
			if (MSXML)
			{
				var DOM = new ActiveXObject('MSXML2.DOMDocument.3.0');
				DOM.async = false;
				DOM.validateOnParse = false;
				DOM.loadXML('<' + elName + '/>');

				return DOM;
			}

			return document.implementation.createDocument('', elName, null);
		}

		var stack = [],
			pos   = 0,
			i     = -1,
			cnt   = processedTags.length,
			DOM   = createDOM((cnt) ? 'rt' : 'pt'),
			el    = DOM.documentElement;

		function writeElement(tagName, content)
		{
			setTextContent(
				el.appendChild(DOM.createElement(tagName)),
				content
			);
		}

		function appendText(content)
		{
			el.appendChild(DOM.createTextNode(content));
		}

		function setTextContent(el, content)
		{
			if (ENABLE_IE_WORKAROUNDS && ENABLE_IE_WORKAROUNDS < 9 && !('textContent' in el))
			{
				el.appendChild(DOM.createTextNode(content));
			}
			else
			{
				el.textContent = content;
			}
		}

		while (++i < cnt)
		{
			var tag = processedTags[i];

			/**
			* Append the text that's between last tag and this one
			*/
			if (tag.pos > pos)
			{
				appendText(text.substr(pos, tag.pos - pos));
			}

			/**
			* Capture the part of the text that belongs to this tag then move the cursor past
			* current tag
			*/
			var tagText = text.substr(tag.pos, tag.len);
			pos = tag.pos + tag.len;

			var wsBefore = '',
				wsAfter  = '';

			if (tag.trimBefore)
			{
				wsBefore = tagText.substr(0, tag.trimBefore);
				tagText  = tagText.substr(tag.trimBefore);
{'fraction_non_alphanumeric': 0.1921193232413179, 'fraction_numerical': 0.009572573463935885, 'mean_word_length': 2.36561153774115, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7229738', 'n_tokens_mistral': 8464, 'n_tokens_neox': 6758, 'n_words': 1413}