<commit_before><?php

/**
 * Converts a string of HTML to BBC
 *
 * @package   ElkArte Forum
 * @copyright ElkArte Forum contributors
 * @license   BSD http://opensource.org/licenses/BSD-3-Clause (see accompanying LICENSE.txt file)
 *
 * @version 2.0 dev
 *
 */

namespace ElkArte;

/**
 * Converts a string of HTML to BBC
 *
 * Initiate
 *    $bbc_converter = new \ElkArte\Html2BBC($html);
 *    where $html is a string of html we want to convert to bbc
 *
 * Override
 *    $bbc_converter->skip_tags(array())
 *    will prevent the conversion of certain html tags to bbc
 *
 * Convert
 *    $bbc = $bbc_converter->get_bbc();
 *
 */
class Html2BBC
{
	/**
	 * The value that will hold our dom object
	 *
	 * @var object
	 */
	public $doc;

	/**
	 * The value that will hold if we are using the internal or external parser
	 *
	 * @var bool
	 */
	private $_parser;

	/**
	 * Line end character
	 *
	 * @var string
	 */
	public $line_end = "\n";

	/**
	 * Line break character
	 *
	 * @var string
	 */
	public $line_break = '[br]';

	/**
	 * Font numbers to pt size
	 *
	 * @var string[]
	 */
	public $sizes_equivalence = array(1 => '8pt', '10pt', '12pt', '14pt', '18pt', '24pt', '36pt');

	/**
	 * Holds block elements, its intentionally not complete and is used to prevent adding extra br's
	 *
	 * @var string[]
	 */
	public $block_elements = array('p', 'div', 'ol', 'ul', 'pre', 'table', 'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6');

	/**
	 * Used to strip newlines inside of 'p' and 'div' elements
	 *
	 * @var bool|null
	 */
	public $strip_newlines = null;

	/**
	 * Holds any html tags that would normally be convert to bbc but are instead skipped
	 *
	 * @var string[]
	 */
	protected $_skip_tags = array();

	/**
	 * Holds any style attributes that would normally be convert to bbc but are instead skipped
	 *
	 * @var string[]
	 */
	protected $_skip_style = array();

	/**
	 * Gets everything started using the built in or external parser
	 *
	 * @param string $html string of html to convert
	 * @param bool $strip flag to strip newlines, true by default
	 */
	public function __construct($html, $strip = true)
	{
		// Up front, remove whitespace between html tags
		$html = preg_replace('/(?:(?<=\>)|(?<=\/\>))(\s+)(?=\<\/?)/', '', $html);
		$html = strtr($html, array('[' => '&amp#91;', ']' => '&amp#93;'));
		$this->strip_newlines = $strip;

		// Using PHP built in functions ...
		if (class_exists('\\DOMDocument'))
		{
			$this->_parser = true;
			$this->doc = new \DOMDocument();
			$this->doc->preserveWhiteSpace = false;

			// Make it a utf-8 doc always and be silent about those html structure errors
			libxml_use_internal_errors(true);
			$this->doc->loadHTML('<?xml encoding="UTF-8">' . $html);
			$this->doc->encoding = 'UTF-8';
			libxml_clear_errors();
		}
		// Or using the external simple html parser
		else
		{
			$this->_parser = false;
			require_once(EXTDIR . '/simple_html_dom.php');
			$this->doc = str_get_html($html, true, true, 'UTF-8', false);
		}
	}

	/**
	 * If we want to skip over some tags (that would normally be converted)
	 *
	 * @param string[] $tags
	 */
	public function skip_tags($tags = array())
	{
		// If its not an array, make it one
		if (!is_array($tags))
		{
			$tags = array($tags);
		}

		if (!empty($tags))
		{
			$this->_skip_tags = $tags;
		}
	}

	/**
	 * If we want to skip over inline style tags (that would normally be converted)
	 *
	 * @param string[] $styles
	 */
	public function skip_styles($styles = array())
	{
		// If its not an array, make it one
		if (!is_array($styles))
		{
			$styles = array($styles);
		}

		if (!empty($styles))
		{
			$this->_skip_style = $styles;
		}
	}

	/**
	 * Loads the html body and sends it to the parsing loop to convert all
	 * DOM nodes to BBC
	 */
	public function get_bbc()
	{
		// For this html node, find all child elements and convert
		$body = ($this->_parser) ? $this->doc->getElementsByTagName('body')->item(0) : $this->doc->root;

		// Convert all the nodes that we know how to
		$this->_convert_childNodes($body);

		// Done replacing HTML elements, now get the converted DOM tree back into a string
		$bbc = ($this->_parser) ? $this->doc->saveHTML() : $this->doc->save();
		$bbc = $this->_recursive_decode($bbc);

		if ($this->_parser)
		{
			// Using the internal DOM methods we need to do a little extra work
			if (preg_match('~<body>(.*)</body>~s', $bbc, $body))
			{
				$bbc = $body[1];
			}
		}

		// Remove comment blocks
		$bbc = preg_replace('~\\<\\!--.*?-->~i', '', $bbc);
		$bbc = preg_replace('~\\<\\!\\[CDATA\\[.*?\\]\\]\\>~i', '', $bbc);

		// Remove non breakable spaces that may be hiding in here
		$bbc = str_replace("\xC2\xA0\x20", ' ', $bbc);
		$bbc = str_replace("\xC2\xA0", ' ', $bbc);

		// Strip any excess leading/trailing blank lines we may have produced O:-)
		$bbc = trim($bbc);
		$bbc = preg_replace('~^(?:\[br\s*\/?\]\s*)+~', '', $bbc);
		$bbc = preg_replace('~(?:\[br\s*\/?\]\s*)+$~', '', $bbc);
		$bbc = preg_replace('~\s?(\[br\])\s?~', '[br]', $bbc);
		$bbc = str_replace('[hr][br]', '[hr]', $bbc);

		// Remove any html tags we left behind ( outside of code tags that is )
		$parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $bbc, -1, PREG_SPLIT_DELIM_CAPTURE);
		foreach ($parts as $i => $part)
		{
			if ($i % 4 === 0)
			{
				// protect << symbols from being stripped
				$working = str_replace('<<', "[\xC2\xA0]", $parts[$i]);
				$working = strip_tags($working);
				$parts[$i] = str_replace("[\xC2\xA0]", '<<', $working);
			}
		}

		return empty($parts) ? '' : implode('', $parts);
	}
<commit_msg>! prevent data loss when false is returned<commit_after><?php

/**
 * Converts a string of HTML to BBC
 *
 * @package   ElkArte Forum
 * @copyright ElkArte Forum contributors
 * @license   BSD http://opensource.org/licenses/BSD-3-Clause (see accompanying LICENSE.txt file)
 *
 * @version 2.0 dev
 *
 */

namespace ElkArte;

/**
 * Converts a string of HTML to BBC
 *
 * Initiate
 *    $bbc_converter = new \ElkArte\Html2BBC($html);
 *    where $html is a string of html we want to convert to bbc
 *
 * Override
 *    $bbc_converter->skip_tags(array())
 *    will prevent the conversion of certain html tags to bbc
 *
 * Convert
 *    $bbc = $bbc_converter->get_bbc();
 *
 */
class Html2BBC
{
	/**
	 * The value that will hold our dom object
	 *
	 * @var object
	 */
	public $doc;

	/**
	 * The value that will hold if we are using the internal or external parser
	 *
	 * @var bool
	 */
	private $_parser;

	/**
	 * Line end character
	 *
	 * @var string
	 */
	public $line_end = "\n";

	/**
	 * Line break character
	 *
	 * @var string
	 */
	public $line_break = '[br]';

	/**
	 * Font numbers to pt size
	 *
	 * @var string[]
	 */
	public $sizes_equivalence = array(1 => '8pt', '10pt', '12pt', '14pt', '18pt', '24pt', '36pt');

	/**
	 * Holds block elements, its intentionally not complete and is used to prevent adding extra br's
	 *
	 * @var string[]
	 */
	public $block_elements = array('p', 'div', 'ol', 'ul', 'pre', 'table', 'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6');

	/**
	 * Used to strip newlines inside of 'p' and 'div' elements
	 *
	 * @var bool|null
	 */
	public $strip_newlines = null;

	/**
	 * Holds any html tags that would normally be convert to bbc but are instead skipped
	 *
	 * @var string[]
	 */
	protected $_skip_tags = array();

	/**
	 * Holds any style attributes that would normally be convert to bbc but are instead skipped
	 *
	 * @var string[]
	 */
	protected $_skip_style = array();

	/**
	 * Gets everything started using the built in or external parser
	 *
	 * @param string $html string of html to convert
	 * @param bool $strip flag to strip newlines, true by default
	 */
	public function __construct($html, $strip = true)
	{
		// Up front, remove whitespace between html tags
		$html = preg_replace('/(?:(?<=\>)|(?<=\/\>))(\s+)(?=\<\/?)/', '', $html);
		$html = strtr($html, array('[' => '&amp#91;', ']' => '&amp#93;'));
		$this->strip_newlines = $strip;

		// Using PHP built in functions ...
		if (class_exists('\\DOMDocument'))
		{
			$this->_parser = true;
			$this->doc = new \DOMDocument();
			$this->doc->preserveWhiteSpace = false;

			// Make it a utf-8 doc always and be silent about those html structure errors
			libxml_use_internal_errors(true);
			$this->doc->loadHTML('<?xml encoding="UTF-8">' . $html);
			$this->doc->encoding = 'UTF-8';
			libxml_clear_errors();
		}
		// Or using the external simple html parser
		else
		{
			$this->_parser = false;
			require_once(EXTDIR . '/simple_html_dom.php');
			$this->doc = str_get_html($html, true, true, 'UTF-8', false);
		}
	}

	/**
	 * If we want to skip over some tags (that would normally be converted)
	 *
	 * @param string[] $tags
	 */
	public function skip_tags($tags = array())
	{
		// If its not an array, make it one
		if (!is_array($tags))
		{
			$tags = array($tags);
		}

		if (!empty($tags))
		{
			$this->_skip_tags = $tags;
		}
	}

	/**
	 * If we want to skip over inline style tags (that would normally be converted)
	 *
	 * @param string[] $styles
	 */
	public function skip_styles($styles = array())
	{
		// If its not an array, make it one
		if (!is_array($styles))
		{
			$styles = array($styles);
		}

		if (!empty($styles))
		{
			$this->_skip_style = $styles;
		}
	}

	/**
	 * Loads the html body and sends it to the parsing loop to convert all
	 * DOM nodes to BBC
	 */
	public function get_bbc()
	{
		// For this html node, find all child elements and convert
		$body = ($this->_parser) ? $this->doc->getElementsByTagName('body')->item(0) : $this->doc->root;

		// Convert all the nodes that we know how to
		$this->_convert_childNodes($body);

		// Done replacing HTML elements, now get the converted DOM tree back into a string
		$bbc = ($this->_parser) ? $this->doc->saveHTML() : $this->doc->save();
		$bbc = $this->_recursive_decode($bbc);

		if ($this->_parser)
		{
			// Using the internal DOM methods we need to do a little extra work
			if (preg_match('~<body>(.*)</body>~s', $bbc, $body))
			{
				$bbc = $body[1];
			}
		}

		// Remove comment blocks
		$bbc = preg_replace('~\\<\\!--.*?-->~i', '', $bbc);
		$bbc = preg_replace('~\\<\\!\\[CDATA\\[.*?\\]\\]\\>~i', '', $bbc);

		// Remove non breakable spaces that may be hiding in here
		$bbc = str_replace("\xC2\xA0\x20", ' ', $bbc);
		$bbc = str_replace("\xC2\xA0", ' ', $bbc);

		// Strip any excess leading/trailing blank lines we may have produced O:-)
		$bbc = trim($bbc);
		$bbc = preg_replace('~^(?:\[br\s*\/?\]\s*)+~', '', $bbc);
		$bbc = preg_replace('~(?:\[br\s*\/?\]\s*)+$~', '', $bbc);
		$bbc = preg_replace('~\s?(\[br\])\s?~', '[br]', $bbc);
		$bbc = str_replace('[hr][br]', '[hr]', $bbc);

		// Remove any html tags we left behind ( outside of code tags that is )
		$parts = preg_split('~(\[/code\]|\[code(?:=[^\]]+)?\])~i', $bbc, -1, PREG_SPLIT_DELIM_CAPTURE);
		if ($parts !== false)
		{
			foreach ($parts as $i => $part)
			{
				if ($i % 4 === 0)
				{
					// protect << symbols from being stripped
					$working = str_replace('<<', "[\xC2\xA0]", $parts[$i]);
					$working = strip_tags($working);
					$parts[$i] = str_replace("[\xC2\xA0]", '<<', $working);
				}
			}
		}

		return empty($parts) ? '' : implode('', $parts);
	}

{'fraction_non_alphanumeric': 0.19022563096405956, 'fraction_numerical': 0.008561491126371177, 'mean_word_length': 3.0513005780346822, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363889', 'n_tokens_mistral': 4349, 'n_tokens_neox': 3893, 'n_words': 1164}