<commit_before><?php
namespace wcf\system\html\input;
use wcf\system\bbcode\HtmlBBCodeParser;
use wcf\system\html\input\filter\IHtmlInputFilter;
use wcf\system\html\input\filter\MessageHtmlInputFilter;
use wcf\system\html\input\node\HtmlInputNodeProcessor;
use wcf\system\html\AbstractHtmlProcessor;
use wcf\util\DOMUtil;
use wcf\util\StringUtil;

/**
 * Reads a HTML string, applies filters and parses all nodes including bbcodes.
 * 
 * @author      <NAME>
 * @copyright   2001-2018 WoltLab GmbH
 * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package     WoltLabSuite\Core\System\Html\Input
 * @since       3.0
 */
class HtmlInputProcessor extends AbstractHtmlProcessor {
	/**
	 * list of embedded content grouped by type
	 * @var array
	 */
	protected $embeddedContent = [];
	
	/**
	 * @var	IHtmlInputFilter
	 */
	protected $htmlInputFilter;
	
	/**
	 * @var HtmlInputNodeProcessor
	 */
	protected $htmlInputNodeProcessor;
	
	/**
	 * Processes the input html string.
	 *
	 * @param       string          $html                   html string
	 * @param       string          $objectType             object type identifier
	 * @param       integer         $objectID               object id
	 * @param       boolean         $convertFromBBCode      interpret input as bbcode
	 */
	public function process($html, $objectType, $objectID = 0, $convertFromBBCode = false) {
		$this->reset();
		
		$this->setContext($objectType, $objectID);
		
		// enforce consistent newlines
		$html = StringUtil::trim(StringUtil::unifyNewlines($html));
		
		// check if this is true HTML or just a bbcode string
		if ($convertFromBBCode) {
			$html = $this->convertToHtml($html);
		}
		
		// transform bbcodes into metacode markers
		$html = HtmlBBCodeParser::getInstance()->parse($html);
		
		// filter HTML
		$html = $this->getHtmlInputFilter()->apply($html);
		
		// pre-parse HTML
		$this->getHtmlInputNodeProcessor()->load($this, $html);
		$this->getHtmlInputNodeProcessor()->process();
		$this->embeddedContent = $this->getHtmlInputNodeProcessor()->getEmbeddedContent();
	}
	
	/**
	 * Processes a HTML string to provide the general DOM API. This method
	 * does not perform any filtering or validation. You SHOULD NOT use this
	 * to deal with HTML that has not been filtered previously.
	 * 
	 * @param       string          $html   html string
	 */
	public function processIntermediate($html) {
		$this->getHtmlInputNodeProcessor()->load($this, $html);
	}
	
	/**
	 * Reprocesses a message by transforming the message into an editor-like
	 * state using plain bbcodes instead of metacode elements.
	 * 
	 * @param       string          $html           html string
	 * @param       string          $objectType     object type identifier
	 * @param       integer         $objectID       object id
	 * @since       3.1
	 */
	public function reprocess($html, $objectType, $objectID) {
		$this->processIntermediate($html);
		
		// revert embedded bbcodes for re-evaluation
		$metacodes = DOMUtil::getElements($this->getHtmlInputNodeProcessor()->getDocument(), 'woltlab-metacode');
		foreach ($metacodes as $metacode) {
			$name = $metacode->getAttribute('data-name');
			$attributes = $this->getHtmlInputNodeProcessor()->parseAttributes($metacode->getAttribute('data-attributes'));
			
			$bbcodeAttributes = '';
			foreach ($attributes as $attribute) {
				if (!empty($bbcodeAttributes)) $bbcodeAttributes .= ',';
				
				if ($attribute === true) $bbcodeAttributes .= 'true';
				else if ($attribute === false) $bbcodeAttributes .= 'false';
				else if (is_string($attribute) || is_numeric($attribute)) {
					$bbcodeAttributes .= "'" . addcslashes($attribute, "'") . "'";
				}
				else {
					// discard anything that is not string-like
					$bbcodeAttributes .= "''";
				}
			}
			
			$text = $metacode->ownerDocument->createTextNode('[' . $name . (!empty($bbcodeAttributes) ? '=' . $bbcodeAttributes : '') . ']');
			$metacode->insertBefore($text, $metacode->firstChild);
			
			$text = $metacode->ownerDocument->createTextNode('[/' . $name . ']');
			$metacode->appendChild($text);
			
			DOMUtil::removeNode($metacode, true);
		}
		
		$this->process($this->getHtml(), $objectType, $objectID, false);
	}
	
	/**
	 * Processes only embedded content. This method should only be called when rebuilding
	 * data where only embedded content is relevant, but no actual parsing is required.
	 * 
	 * @param       string          $html           html string
	 * @param       string          $objectType     object type identifier
	 * @param       integer         $objectID       object id
	 * @throws      \UnexpectedValueException
	 */<commit_msg>Skip the HTML filter when reprocessing messages<commit_after><?php
namespace wcf\system\html\input;
use wcf\system\bbcode\HtmlBBCodeParser;
use wcf\system\html\input\filter\IHtmlInputFilter;
use wcf\system\html\input\filter\MessageHtmlInputFilter;
use wcf\system\html\input\node\HtmlInputNodeProcessor;
use wcf\system\html\AbstractHtmlProcessor;
use wcf\util\DOMUtil;
use wcf\util\StringUtil;

/**
 * Reads a HTML string, applies filters and parses all nodes including bbcodes.
 * 
 * @author      <NAME>
 * @copyright   2001-2018 WoltLab GmbH
 * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package     WoltLabSuite\Core\System\Html\Input
 * @since       3.0
 */
class HtmlInputProcessor extends AbstractHtmlProcessor {
	/**
	 * list of embedded content grouped by type
	 * @var array
	 */
	protected $embeddedContent = [];
	
	/**
	 * @var	IHtmlInputFilter
	 */
	protected $htmlInputFilter;
	
	/**
	 * @var HtmlInputNodeProcessor
	 */
	protected $htmlInputNodeProcessor;
	
	/**
	 * skip the HTML filter during message reprocessing
	 * @var boolean
	 */
	protected $skipFilter = false;
	
	/**
	 * Processes the input html string.
	 *
	 * @param       string          $html                   html string
	 * @param       string          $objectType             object type identifier
	 * @param       integer         $objectID               object id
	 * @param       boolean         $convertFromBBCode      interpret input as bbcode
	 */
	public function process($html, $objectType, $objectID = 0, $convertFromBBCode = false) {
		$this->reset();
		
		$this->setContext($objectType, $objectID);
		
		// enforce consistent newlines
		$html = StringUtil::trim(StringUtil::unifyNewlines($html));
		
		// check if this is true HTML or just a bbcode string
		if ($convertFromBBCode) {
			$html = $this->convertToHtml($html);
		}
		
		// transform bbcodes into metacode markers
		$html = HtmlBBCodeParser::getInstance()->parse($html);
		
		// filter HTML
		if (!$this->skipFilter) {
			$html = $this->getHtmlInputFilter()->apply($html);
		}
		
		// pre-parse HTML
		$this->getHtmlInputNodeProcessor()->load($this, $html);
		$this->getHtmlInputNodeProcessor()->process();
		$this->embeddedContent = $this->getHtmlInputNodeProcessor()->getEmbeddedContent();
	}
	
	/**
	 * Processes a HTML string to provide the general DOM API. This method
	 * does not perform any filtering or validation. You SHOULD NOT use this
	 * to deal with HTML that has not been filtered previously.
	 * 
	 * @param       string          $html   html string
	 */
	public function processIntermediate($html) {
		$this->getHtmlInputNodeProcessor()->load($this, $html);
	}
	
	/**
	 * Reprocesses a message by transforming the message into an editor-like
	 * state using plain bbcodes instead of metacode elements.
	 * 
	 * @param       string          $html           html string
	 * @param       string          $objectType     object type identifier
	 * @param       integer         $objectID       object id
	 * @since       3.1
	 */
	public function reprocess($html, $objectType, $objectID) {
		$this->processIntermediate($html);
		
		// revert embedded bbcodes for re-evaluation
		$metacodes = DOMUtil::getElements($this->getHtmlInputNodeProcessor()->getDocument(), 'woltlab-metacode');
		foreach ($metacodes as $metacode) {
			$name = $metacode->getAttribute('data-name');
			$attributes = $this->getHtmlInputNodeProcessor()->parseAttributes($metacode->getAttribute('data-attributes'));
			
			$bbcodeAttributes = '';
			foreach ($attributes as $attribute) {
				if (!empty($bbcodeAttributes)) $bbcodeAttributes .= ',';
				
				if ($attribute === true) $bbcodeAttributes .= 'true';
				else if ($attribute === false) $bbcodeAttributes .= 'false';
				else if (is_string($attribute) || is_numeric($attribute)) {
					$bbcodeAttributes .= "'" . addcslashes($attribute, "'") . "'";
				}
				else {
					// discard anything that is not string-like
					$bbcodeAttributes .= "''";
				}
			}
			
			$text = $metacode->ownerDocument->createTextNode('[' . $name . (!empty($bbcodeAttributes) ? '=' . $bbcodeAttributes : '') . ']');
			$metacode->insertBefore($text, $metacode->firstChild);
			
			$text = $metacode->ownerDocument->createTextNode('[/' . $name . ']');
			$metacode->appendChild($text);
			
			DOMUtil::removeNode($metacode, true);
		}
		
		try {
			$this->skipFilter = true;
			$this->process($this->getHtml(), $objectType, $objectID, false);
		}
		finally {
			$this->skipFilter = false;
		}
	}
	
	/**
	 * Processes only embedded content. This method should only be called when rebuilding
	 * data where only embedded content is relevant, but no actual parsing is required.
	 * 
	 * @param       string          $html           html string
	 * @param       string          $objectType     object type identifier
	 * @param       integer         $objectID       object id
	 * @throws      \UnexpectedValueException
	 */
{'fraction_non_alphanumeric': 0.12773684756667011, 'fraction_numerical': 0.0026979350420255266, 'mean_word_length': 3.044481745698699, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6442068', 'n_tokens_mistral': 3096, 'n_tokens_neox': 2695, 'n_words': 708}