<commit_before>/**#@+
* Boolean rules bitfield
*/
/** @const */ var RULE_AUTO_CLOSE        =   1;
/** @const */ var RULE_AUTO_REOPEN       =   2;
/** @const */ var RULE_BREAK_PARAGRAPH   =   4;
/** @const */ var RULE_CREATE_PARAGRAPHS =   8;
/** @const */ var RULE_IGNORE_TEXT       =  16;
/** @const */ var RULE_IS_TRANSPARENT    =  32;
/** @const */ var RULE_NO_BR_CHILD       =  64;
/** @const */ var RULE_NO_BR_DESCENDANT  = 128;
/** @const */ var RULE_TRIM_WHITESPACE   = 256;
/**#@-*/

/**
* @const All the characters that are considered whitespace
*/
var WHITESPACE = " \n\t";

/**
* @type {!Logger} This parser's logger
*/
var logger = new Logger;

/**
* @type {!Object} Variables registered for use in filters
*/
var registeredVars;

/**
* @type {!Object} Tags' config
* @const
*/
var tagsConfig;

/**
* @type {!string} Text being parsed
*/
var text;

/**
* @type {!number} Length of the text being parsed
*/
var textLen;

/**
* @type {!number} Random number generated everytime the parser is reset
*/
var uid;

//==========================================================================
// Public API
//==========================================================================

/**
* Disable a tag
*
* @param {!string} tagName Name of the tag
*/
function disableTag(tagName)
{
	if (tagsConfig[tagName])
	{
		tagsConfig[tagName].isDisabled = true;
	}
}

/**
* Enable a tag
*
* @param {!string} tagName Name of the tag
*/
function enableTag(tagName)
{
	if (tagsConfig[tagName])
	{
		tagsConfig[tagName].isDisabled = false;
	}
}

/**
* Get this parser's Logger instance
*
* @return {!Logger}
*/
function getLogger()
{
	return logger;
}

/**
* Parse a text
*
* @param  {!string} _text Text to parse
* @return {!string}       XML representation
*/
function parse(_text)
{
	// Reset the parser and save the uid
	reset(_text);
	var _uid = +uid;

	// Do the heavy lifting<commit_msg>Removed Closure Compiler workaround. See http://code.google.com/p/closure-compiler/issues/detail?id=1053<commit_after>/**#@+
* Boolean rules bitfield
*/
/** @const */ var RULE_AUTO_CLOSE        =   1;
/** @const */ var RULE_AUTO_REOPEN       =   2;
/** @const */ var RULE_BREAK_PARAGRAPH   =   4;
/** @const */ var RULE_CREATE_PARAGRAPHS =   8;
/** @const */ var RULE_IGNORE_TEXT       =  16;
/** @const */ var RULE_IS_TRANSPARENT    =  32;
/** @const */ var RULE_NO_BR_CHILD       =  64;
/** @const */ var RULE_NO_BR_DESCENDANT  = 128;
/** @const */ var RULE_TRIM_WHITESPACE   = 256;
/**#@-*/

/**
* @const All the characters that are considered whitespace
*/
var WHITESPACE = " \n\t";

/**
* @type {!Logger} This parser's logger
*/
var logger = new Logger;

/**
* @type {!Object} Variables registered for use in filters
*/
var registeredVars;

/**
* @type {!Object} Tags' config
* @const
*/
var tagsConfig;

/**
* @type {!string} Text being parsed
*/
var text;

/**
* @type {!number} Length of the text being parsed
*/
var textLen;

/**
* @type {!number} Random number generated everytime the parser is reset
*/
var uid;

//==========================================================================
// Public API
//==========================================================================

/**
* Disable a tag
*
* @param {!string} tagName Name of the tag
*/
function disableTag(tagName)
{
	if (tagsConfig[tagName])
	{
		tagsConfig[tagName].isDisabled = true;
	}
}

/**
* Enable a tag
*
* @param {!string} tagName Name of the tag
*/
function enableTag(tagName)
{
	if (tagsConfig[tagName])
	{
		tagsConfig[tagName].isDisabled = false;
	}
}

/**
* Get this parser's Logger instance
*
* @return {!Logger}
*/
function getLogger()
{
	return logger;
}

/**
* Parse a text
*
* @param  {!string} _text Text to parse
* @return {!string}       XML representation
*/
function parse(_text)
{
	// Reset the parser and save the uid
	reset(_text);
	var _uid = uid;

	// Do the heavy lifting
{'fraction_non_alphanumeric': 0.2267080745341615, 'fraction_numerical': 0.009316770186335404, 'mean_word_length': 3.5902612826603324, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1662914', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1309, 'n_words': 366}