<commit_before><?php
namespace wcf\system\bbcode;
use wcf\system\exception\SystemException;
use wcf\util\JSON;
use wcf\util\StringUtil;

/**
 * Parses bbcodes and transforms them into the custom HTML element <woltlab-bbcode>
 * that can be safely passed through HTMLPurifier's validation mechanism.
 * 
 * All though not exactly required for all bbcodes, the actual output of an bbcode
 * cannot be foreseen and potentially conflict with HTMLPurifier's whitelist. Examples
 * are <iframe> or other embedded media that is allowed as a result of a bbcode, but
 * not allowed to be directly provided by a user. 
 * 
 * @author	<NAME>
 * @copyright	2001-2016 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	WoltLabSuite\Core\System\Bbcode
 * @since	3.0
 */
class HtmlBBCodeParser extends BBCodeParser {
	/**
	 * list of open tags with name and uuid
	 * @var array
	 */
	protected $openTagIdentifiers = [];
	
	/**
	 * regex for valid bbcode names
	 * @var string
	 */
	protected $validBBCodePattern = '~^[a-z](?:[a-z0-9\-_]+)?$~';
	
	/**
	 * @inheritDoc
	 */
	public function parse($text) {
		$this->setText($text);
		
		// difference to the original implementation: sourcecode bbcodes are handled too
		$this->buildTagArray(false);
		
		$this->ignoreUnclosedTags();
		
		$this->buildXMLStructure();
		
		$this->buildParsedString();
		
		return $this->parsedText;
	}
	
	/**
	 * Reverts tags to their source representation if they either
	 * have no matching counter part (such as opening tags without
	 * closing one), or if they're inside code bbcodes.
	 */
	protected function ignoreUnclosedTags() {
		$length = count($this->tagArray);
		
		// step 1) validate source bbcodes
		$inSource = null;
		$sourceBBCodes = $this->getSourceBBCodes();
		foreach ($this->tagArray as $i => &$tag) {
			$name = $tag['name'];
			$tag['valid'] = true;
			
			if ($tag['closing']) {
				if ($inSource === null) {
					continue;
				}
				
				if ($name === $inSource['name']) {
					$inSource = null;
				}
				else {
					$tag['valid'] = false;
				}
			}
			else {
				if ($inSource !== null) {
					$tag['valid'] = false;
					continue;
				}
				
				if (in_array($name, $sourceBBCodes)) {
					// look ahead to see if there is a closing tag
					$hasClosingTag = false;
					for ($j = $i + 1; $j < $length; $j++) {
						if ($this->tagArray[$j]['closing'] && $this->tagArray[$j]['name'] === $name) {
							$hasClosingTag = true;
							break;
						}
					}
					
					if ($hasClosingTag) {
						$inSource = $tag;
					}
					else {
						$tag['valid'] = false;
					}
				}
			}
		}
		unset($tag);
		
		// step 2) check if tags are properly opened and closed, incorrect nesting doesn't matter here
		foreach ($this->tagArray as $i => &$tag) {
			if (!$tag['valid']) {
				continue;
			}
			
			if ($tag['closing']) {
				if (!isset($tag['matching'])) {
					$tag['valid'] = false;
				}
			}
			else {
				$name = $tag['name'];
				
				// find matching closing tag
				$hasClosingTag = false;
				$badTags = [];
				for ($j = $i + 1; $j < $length; $j++) {
					$sibling = $this->tagArray[$j];
					if ($sibling['name'] === $name) {
						if (!$sibling['closing']) {
							if ($name === 'quote') {
								continue;
							}
							
							// disallow the same tag opened again
							$badTags[] = $j;
						}
						else if (!isset($sibling['matching'])) {
							$this->tagArray[$j]['matching'] = true;
							$hasClosingTag = true;
							break;
						}
					}
				}
				
				if ($hasClosingTag) {
					foreach ($badTags as $j) {
						$this->tagArray[$j]['valid'] = false;
					}
				}
				else {
					$tag['valid'] = false;
				}<commit_msg>Fixed nesting of certain bbcodes<commit_after><?php
namespace wcf\system\bbcode;
use wcf\system\exception\SystemException;
use wcf\util\JSON;
use wcf\util\StringUtil;

/**
 * Parses bbcodes and transforms them into the custom HTML element <woltlab-bbcode>
 * that can be safely passed through HTMLPurifier's validation mechanism.
 * 
 * All though not exactly required for all bbcodes, the actual output of an bbcode
 * cannot be foreseen and potentially conflict with HTMLPurifier's whitelist. Examples
 * are <iframe> or other embedded media that is allowed as a result of a bbcode, but
 * not allowed to be directly provided by a user. 
 * 
 * @author	<NAME>
 * @copyright	2001-2016 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	WoltLabSuite\Core\System\Bbcode
 * @since	3.0
 */
class HtmlBBCodeParser extends BBCodeParser {
	/**
	 * list of bbcodes that cannot be nested
	 * @var string[]
	 */
	public static $disallowNesting = ['attach', 'b', 'code', 'email', 'i', 'img', 'media', 's', 'tt', 'u', 'url', 'user', 'wsm', 'wsp'];
	
	/**
	 * list of open tags with name and uuid
	 * @var array
	 */
	protected $openTagIdentifiers = [];
	
	/**
	 * regex for valid bbcode names
	 * @var string
	 */
	protected $validBBCodePattern = '~^[a-z](?:[a-z0-9\-_]+)?$~';
	
	/**
	 * @inheritDoc
	 */
	public function parse($text) {
		$this->setText($text);
		
		// difference to the original implementation: sourcecode bbcodes are handled too
		$this->buildTagArray(false);
		
		$this->ignoreUnclosedTags();
		
		$this->buildXMLStructure();
		
		$this->buildParsedString();
		
		return $this->parsedText;
	}
	
	/**
	 * Reverts tags to their source representation if they either
	 * have no matching counter part (such as opening tags without
	 * closing one), or if they're inside code bbcodes.
	 */
	protected function ignoreUnclosedTags() {
		$length = count($this->tagArray);
		
		// step 1) validate source bbcodes
		$inSource = null;
		$sourceBBCodes = $this->getSourceBBCodes();
		foreach ($this->tagArray as $i => &$tag) {
			$name = $tag['name'];
			$tag['valid'] = true;
			
			if ($tag['closing']) {
				if ($inSource === null) {
					continue;
				}
				
				if ($name === $inSource['name']) {
					$inSource = null;
				}
				else {
					$tag['valid'] = false;
				}
			}
			else {
				if ($inSource !== null) {
					$tag['valid'] = false;
					continue;
				}
				
				if (in_array($name, $sourceBBCodes)) {
					// look ahead to see if there is a closing tag
					$hasClosingTag = false;
					for ($j = $i + 1; $j < $length; $j++) {
						if ($this->tagArray[$j]['closing'] && $this->tagArray[$j]['name'] === $name) {
							$hasClosingTag = true;
							break;
						}
					}
					
					if ($hasClosingTag) {
						$inSource = $tag;
					}
					else {
						$tag['valid'] = false;
					}
				}
			}
		}
		unset($tag);
		
		// step 2) check if tags are properly opened and closed, incorrect nesting doesn't matter here
		foreach ($this->tagArray as $i => &$tag) {
			if (!$tag['valid']) {
				continue;
			}
			
			if ($tag['closing']) {
				if (!isset($tag['matching'])) {
					$tag['valid'] = false;
				}
			}
			else {
				$name = $tag['name'];
				
				// find matching closing tag
				$hasClosingTag = false;
				$badTags = [];
				for ($j = $i + 1; $j < $length; $j++) {
					$sibling = $this->tagArray[$j];
					if ($sibling['name'] === $name) {
						if (!$sibling['closing']) {
							if (!in_array($name, self::$disallowNesting)) {
								continue;
							}
							
							// disallow the same tag opened again
							$badTags[] = $j;
						}
						else if (!isset($sibling['matching'])) {
							$this->tagArray[$j]['matching'] = true;
							$hasClosingTag = true;
							break;
						}
					}
				}
				
				if ($hasClosingTag) {
					foreach ($badTags as $j) {
						$this->tagArray[$j]['valid'] = false;
					}
				}
				else {
					$tag['valid'] = false;
				}
{'fraction_non_alphanumeric': 0.15866388308977036, 'fraction_numerical': 0.0041753653444676405, 'mean_word_length': 2.627543776620918, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2370619', 'n_tokens_mistral': 3267, 'n_tokens_neox': 2460, 'n_words': 701}