<commit_before><?php
namespace wcf\system\html\output\node;
use wcf\system\bbcode\BBCodeHandler;
use wcf\system\event\EventHandler;
use wcf\system\html\node\AbstractHtmlNodeProcessor;
use wcf\system\Regex;
use wcf\system\WCF;
use wcf\util\StringUtil;

/**
 * Processes code listings.
 * 
 * @author      <NAME>
 * @copyright	2001-2019 WoltLab GmbH
 * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package     WoltLabSuite\Core\System\Html\Output\Node
 * @since       3.0
 */
class HtmlOutputNodePre extends AbstractHtmlOutputNode {
	/**
	 * @inheritDoc
	 */
	protected $tagName = 'pre';
	
	/**
	 * already used ids for line numbers to prevent duplicate ids in the output
	 * @var	string[]
	 */
	private static $codeIDs = [];
	
	/**
	 * @inheritDoc
	 */
	public function process(array $elements, AbstractHtmlNodeProcessor $htmlNodeProcessor) {
		/** @var \DOMElement $element */
		foreach ($elements as $element) {
			if ($element->getAttribute('class') === 'woltlabHtml') {
				$nodeIdentifier = StringUtil::getRandomID();
				$htmlNodeProcessor->addNodeData($this, $nodeIdentifier, ['rawHTML' => $element->textContent]);
				
				$htmlNodeProcessor->renameTag($element, 'wcfNode-' . $nodeIdentifier);
				continue;
			}
			
			switch ($this->outputType) {
				case 'text/html':
					$nodeIdentifier = StringUtil::getRandomID();
					$context = $htmlNodeProcessor->getHtmlProcessor()->getContext();
					$prefix = '';
					// Create a unique prefix if possible
					if (isset($context['objectType']) && isset($context['objectID'])) {
						$prefix = $context['objectType'].'_'.$context['objectID'].'_';
					}
					$htmlNodeProcessor->addNodeData($this, $nodeIdentifier, [
						'content' => $element->textContent,
						'file' => $element->getAttribute('data-file'),
						'highlighter' => $element->getAttribute('data-highlighter'),
						'line' => $element->hasAttribute('data-line') ? $element->getAttribute('data-line') : 1,
						'skipInnerContent' => true,
						'prefix' => $prefix
					]);
					
					$htmlNodeProcessor->renameTag($element, 'wcfNode-' . $nodeIdentifier);
					break;
				
				case 'text/simplified-html':
				case 'text/plain':
					$htmlNodeProcessor->replaceElementWithText(
						$element,
						WCF::getLanguage()->getDynamicVariable('wcf.bbcode.code.simplified', ['lines' => substr_count($element->nodeValue, "\n") + 1]),
						true
					);
					break;
			}
		}
	}
	
	/**
	 * @inheritDoc
	 */
	public function replaceTag(array $data) {
		// HTML bbcode
		if (isset($data['rawHTML'])) {
			return $data['rawHTML'];
		}
		
		$content = preg_replace('/^\s*\n/', '', $data['content']);
		$content = preg_replace('/\n\s*$/', '', $content);
		
		$file = $data['file'];
		$highlighter = $data['highlighter'];
		$line = ($data['line'] < 1) ? 1 : $data['line'];
		
		switch ($highlighter) {
			case 'js':
				$highlighter = 'javascript';
				break;
			case 'c++':
				$highlighter = 'cpp';
				break;
			case 'tex':
				$highlighter = 'latex';
				break;
			case 'shell':
				$highlighter = 'bash';
				break;
		}
		
		if (!$highlighter) {
			// try to guess highlighter
			if (mb_strpos($content, '<?php') !== false) {
				$highlighter = 'php';
			}
			else if (mb_strpos($content, '<html') !== false) {
				$highlighter = 'html';
			}
			else if (mb_strpos($content, '<?xml') === 0) {
				$highlighter = 'xml';
			}
			else if (	mb_strpos($content, 'SELECT') === 0
				||	mb_strpos($content, 'UPDATE') === 0
				||	mb_strpos($content, 'INSERT') === 0
				||	mb_strpos($content, 'DELETE') === 0) {
				$highlighter = 'sql';
			}
			else if (mb_strpos($content, 'import java.') !== false) {
				$highlighter = 'java';
			}
			else if (	mb_strpos($content, "---") !== false
				&&	mb_strpos($content, "\n+++") !== false) {
				$highlighter = 'diff';
			}
			else if (mb_strpos($content, "\n#include ") !== false) {
				$highlighter = 'c';
			}
			else if (mb_strpos($content, '#!/usr/bin/perl') === 0) {
				$highlighter = 'perl';
			}
			else if (mb_strpos($content, 'def __init__(self') !== false) {
				$highlighter = 'python';
			}
			else if (Regex::compile('^#!/bin/(ba|z)?sh')->match($content)) {
				$highlighter = 'bash';
			}
			else if (mb_strpos($content, '\\documentclass') !== false) {
				$highlighter = 'latex';
			}
			else if (mb_strpos($content, '!important;') !== false) {
				$highlighter = 'css';
			}
		}
		$eventData = [
			'highlighter' => $highlighter,
			'data' => $data,
			'content' => $content
		];
		EventHandler::getInstance()->fireAction($this, 'selectHighlighter', $eventData);
		$highlighter = $eventData['highlighter'];
		
		$meta = BBCodeHandler::getInstance()->getHighlighterMeta();
		$title = WCF::getLanguage()->get('wcf.bbcode.code');
		if (isset($meta[$highlighter])) {
			$title = $meta[$highlighter]['title'];
		}
		else {
			$highlighter = null;
		}

		$splitContent = explode("\n", $content);
		$last = array_pop($splitContent);
		$splitContent = array_map(function ($item) {
			return $item."\n";
		}, $splitContent);
		$splitContent[] = $last;
		
		// show template
		/** @noinspection PhpUndefinedMethodInspection */
		WCF::getTPL()->assign([
			'codeID' => $this->getCodeID($data['prefix'] ?? '', $content),
			'startLineNumber' => $line,
			'content' => $splitContent,
			'language' => $highlighter,
			'filename' => $file,
			'title' => $title,
			'lines' => count($splitContent)
		]);
		
		return WCF::getTPL()->fetch('codeMetaCode');
	}
	
	/**
	 * Returns a unique ID for this code block.
	 *
	 * @param	string		$prefix
	 * @param	string		$code
	 * @return	string
	 */
	protected function getCodeID($prefix, $code) {
		$i = -1;
		// find an unused codeID
		do {
			$codeID = $prefix.mb_substr(StringUtil::getHash($code), 0, 6).(++$i ? '_'.$i : '');
		}
		while (isset(self::$codeIDs[$codeID]));
		
		// mark codeID as used
		self::$codeIDs[$codeID] = true;
		
		return $codeID;
	}
}
<commit_msg>Avoid dots in IDs, they don't play well<commit_after><?php
namespace wcf\system\html\output\node;
use wcf\system\bbcode\BBCodeHandler;
use wcf\system\event\EventHandler;
use wcf\system\html\node\AbstractHtmlNodeProcessor;
use wcf\system\Regex;
use wcf\system\WCF;
use wcf\util\StringUtil;

/**
 * Processes code listings.
 * 
 * @author      <NAME>
 * @copyright	2001-2019 WoltLab GmbH
 * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package     WoltLabSuite\Core\System\Html\Output\Node
 * @since       3.0
 */
class HtmlOutputNodePre extends AbstractHtmlOutputNode {
	/**
	 * @inheritDoc
	 */
	protected $tagName = 'pre';
	
	/**
	 * already used ids for line numbers to prevent duplicate ids in the output
	 * @var	string[]
	 */
	private static $codeIDs = [];
	
	/**
	 * @inheritDoc
	 */
	public function process(array $elements, AbstractHtmlNodeProcessor $htmlNodeProcessor) {
		/** @var \DOMElement $element */
		foreach ($elements as $element) {
			if ($element->getAttribute('class') === 'woltlabHtml') {
				$nodeIdentifier = StringUtil::getRandomID();
				$htmlNodeProcessor->addNodeData($this, $nodeIdentifier, ['rawHTML' => $element->textContent]);
				
				$htmlNodeProcessor->renameTag($element, 'wcfNode-' . $nodeIdentifier);
				continue;
			}
			
			switch ($this->outputType) {
				case 'text/html':
					$nodeIdentifier = StringUtil::getRandomID();
					$context = $htmlNodeProcessor->getHtmlProcessor()->getContext();
					$prefix = '';
					// Create a unique prefix if possible
					if (isset($context['objectType']) && isset($context['objectID'])) {
						$prefix = str_replace('.', '_', $context['objectType']).'_'.$context['objectID'].'_';
					}
					$htmlNodeProcessor->addNodeData($this, $nodeIdentifier, [
						'content' => $element->textContent,
						'file' => $element->getAttribute('data-file'),
						'highlighter' => $element->getAttribute('data-highlighter'),
						'line' => $element->hasAttribute('data-line') ? $element->getAttribute('data-line') : 1,
						'skipInnerContent' => true,
						'prefix' => $prefix
					]);
					
					$htmlNodeProcessor->renameTag($element, 'wcfNode-' . $nodeIdentifier);
					break;
				
				case 'text/simplified-html':
				case 'text/plain':
					$htmlNodeProcessor->replaceElementWithText(
						$element,
						WCF::getLanguage()->getDynamicVariable('wcf.bbcode.code.simplified', ['lines' => substr_count($element->nodeValue, "\n") + 1]),
						true
					);
					break;
			}
		}
	}
	
	/**
	 * @inheritDoc
	 */
	public function replaceTag(array $data) {
		// HTML bbcode
		if (isset($data['rawHTML'])) {
			return $data['rawHTML'];
		}
		
		$content = preg_replace('/^\s*\n/', '', $data['content']);
		$content = preg_replace('/\n\s*$/', '', $content);
		
		$file = $data['file'];
		$highlighter = $data['highlighter'];
		$line = ($data['line'] < 1) ? 1 : $data['line'];
		
		switch ($highlighter) {
			case 'js':
				$highlighter = 'javascript';
				break;
			case 'c++':
				$highlighter = 'cpp';
				break;
			case 'tex':
				$highlighter = 'latex';
				break;
			case 'shell':
				$highlighter = 'bash';
				break;
		}
		
		if (!$highlighter) {
			// try to guess highlighter
			if (mb_strpos($content, '<?php') !== false) {
				$highlighter = 'php';
			}
			else if (mb_strpos($content, '<html') !== false) {
				$highlighter = 'html';
			}
			else if (mb_strpos($content, '<?xml') === 0) {
				$highlighter = 'xml';
			}
			else if (	mb_strpos($content, 'SELECT') === 0
				||	mb_strpos($content, 'UPDATE') === 0
				||	mb_strpos($content, 'INSERT') === 0
				||	mb_strpos($content, 'DELETE') === 0) {
				$highlighter = 'sql';
			}
			else if (mb_strpos($content, 'import java.') !== false) {
				$highlighter = 'java';
			}
			else if (	mb_strpos($content, "---") !== false
				&&	mb_strpos($content, "\n+++") !== false) {
				$highlighter = 'diff';
			}
			else if (mb_strpos($content, "\n#include ") !== false) {
				$highlighter = 'c';
			}
			else if (mb_strpos($content, '#!/usr/bin/perl') === 0) {
				$highlighter = 'perl';
			}
			else if (mb_strpos($content, 'def __init__(self') !== false) {
				$highlighter = 'python';
			}
			else if (Regex::compile('^#!/bin/(ba|z)?sh')->match($content)) {
				$highlighter = 'bash';
			}
			else if (mb_strpos($content, '\\documentclass') !== false) {
				$highlighter = 'latex';
			}
			else if (mb_strpos($content, '!important;') !== false) {
				$highlighter = 'css';
			}
		}
		$eventData = [
			'highlighter' => $highlighter,
			'data' => $data,
			'content' => $content
		];
		EventHandler::getInstance()->fireAction($this, 'selectHighlighter', $eventData);
		$highlighter = $eventData['highlighter'];
		
		$meta = BBCodeHandler::getInstance()->getHighlighterMeta();
		$title = WCF::getLanguage()->get('wcf.bbcode.code');
		if (isset($meta[$highlighter])) {
			$title = $meta[$highlighter]['title'];
		}
		else {
			$highlighter = null;
		}

		$splitContent = explode("\n", $content);
		$last = array_pop($splitContent);
		$splitContent = array_map(function ($item) {
			return $item."\n";
		}, $splitContent);
		$splitContent[] = $last;
		
		// show template
		/** @noinspection PhpUndefinedMethodInspection */
		WCF::getTPL()->assign([
			'codeID' => $this->getCodeID($data['prefix'] ?? '', $content),
			'startLineNumber' => $line,
			'content' => $splitContent,
			'language' => $highlighter,
			'filename' => $file,
			'title' => $title,
			'lines' => count($splitContent)
		]);
		
		return WCF::getTPL()->fetch('codeMetaCode');
	}
	
	/**
	 * Returns a unique ID for this code block.
	 *
	 * @param	string		$prefix
	 * @param	string		$code
	 * @return	string
	 */
	protected function getCodeID($prefix, $code) {
		$i = -1;
		// find an unused codeID
		do {
			$codeID = $prefix.mb_substr(StringUtil::getHash($code), 0, 6).(++$i ? '_'.$i : '');
		}
		while (isset(self::$codeIDs[$codeID]));
		
		// mark codeID as used
		self::$codeIDs[$codeID] = true;
		
		return $codeID;
	}
}

{'fraction_non_alphanumeric': 0.1958988150264728, 'fraction_numerical': 0.003865871081603496, 'mean_word_length': 3.4353335818114052, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3231442', 'n_tokens_mistral': 4887, 'n_tokens_neox': 4036, 'n_words': 786}