<commit_before><?php

/**
* @package   s9e\Toolkit
* @copyright Copyright (c) 2010-2011 The s9e Authors
* @license   http://www.opensource.org/licenses/mit-license.php The MIT License
*/
namespace s9e\Toolkit\TextFormatter;

use RuntimeException,
    XMLWriter;

class Parser
{
	/**
	* Start tag, e.g. [b]
	* -- becomes <B><st>[b]</st>
	*/
	const START_TAG = 1;

	/**
	* End tag, e.g. [/b]
	* -- becomes <et>[/b]</et></B>
	*/
	const END_TAG = 2;

	/**
	* Self-closing tag, e.g. [img="http://..." /]
	* -- becomes <IMG>[img="http://..." /]</IMG>
	*
	* NOTE: SELF_CLOSING_TAG = START_TAG | END_TAG
	*/
	const SELF_CLOSING_TAG = 3;

	/**
	* Characters that are removed by the trim_* config directives
	* @link http://docs.php.net/manual/en/function.trim.php
	*/
	const TRIM_CHARLIST = " \n\r\t\0\x0B";

	//==============================================================================================
	// Application stuff
	//==============================================================================================

	/**
	* @var array Logged messages, reinitialized whenever a text is parsed
	*/
	protected $log = array();

	/**
	* @var array Tags config
	*/
	protected $tagsConfig;

	/**
	* @var array Plugins config
	*/
	protected $pluginsConfig;

	/**
	* @var array Filters config
	*/
	protected $filtersConfig;

	/**
	* @var array Array of PluginParser instances
	*/
	protected $pluginParsers = array();

	//==============================================================================================
	// Per-formatting vars
	//==============================================================================================

	/**
	* @var string Text being parsed
	*/
	protected $text;

	/**
	* @var integer Cursor position during parsing
	*/
	protected $pos;

	/**
	* @var array  Unprocessed tags, in reverse order
	*/
	protected $unprocessedTags;

	/**
	* @var array  Processed tags, in document order
	*/
	protected $processedTags;

	/**
	* @var array  Tag currently being processed, used in processTags()
	*/
	protected $currentTag;

	/**
	* @var string Name of the attribute currently being validated, used in processTags()
	*/
	protected $currentAttribute;
<commit_msg>TextFormatter: whitespace<commit_after><?php

/**
* @package   s9e\Toolkit
* @copyright Copyright (c) 2010-2011 The s9e Authors
* @license   http://www.opensource.org/licenses/mit-license.php The MIT License
*/
namespace s9e\Toolkit\TextFormatter;

use RuntimeException,
    XMLWriter;

class Parser
{
	/**
	* Start tag, e.g. [b]
	* -- becomes <B><st>[b]</st>
	*/
	const START_TAG = 1;

	/**
	* End tag, e.g. [/b]
	* -- becomes <et>[/b]</et></B>
	*/
	const END_TAG = 2;

	/**
	* Self-closing tag, e.g. [img="http://..." /]
	* -- becomes <IMG>[img="http://..." /]</IMG>
	*
	* NOTE: SELF_CLOSING_TAG = START_TAG | END_TAG
	*/
	const SELF_CLOSING_TAG = 3;

	/**
	* Characters that are removed by the trim_* config directives
	* @link http://docs.php.net/manual/en/function.trim.php
	*/
	const TRIM_CHARLIST = " \n\r\t\0\x0B";

	//==============================================================================================
	// Application stuff
	//==============================================================================================

	/**
	* @var array Logged messages, reinitialized whenever a text is parsed
	*/
	protected $log = array();

	/**
	* @var array Tags config
	*/
	protected $tagsConfig;

	/**
	* @var array Plugins config
	*/
	protected $pluginsConfig;

	/**
	* @var array Filters config
	*/
	protected $filtersConfig;

	/**
	* @var array Array of PluginParser instances
	*/
	protected $pluginParsers = array();

	//==============================================================================================
	// Per-formatting vars
	//==============================================================================================

	/**
	* @var string  Text being parsed
	*/
	protected $text;

	/**
	* @var integer Cursor position during parsing
	*/
	protected $pos;

	/**
	* @var array   Unprocessed tags, in reverse order
	*/
	protected $unprocessedTags;

	/**
	* @var array   Processed tags, in document order
	*/
	protected $processedTags;

	/**
	* @var array   Tag currently being processed, used in processTags()
	*/
	protected $currentTag;

	/**
	* @var string  Name of the attribute currently being validated, used in processTags()
	*/
	protected $currentAttribute;

{'fraction_non_alphanumeric': 0.30518146541885416, 'fraction_numerical': 0.007304268431864871, 'mean_word_length': 3.8313120176405735, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5368457', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1316, 'n_words': 341}