<commit_before><?php
namespace wcf\system\request;
use wcf\data\page\PageCache;
use wcf\data\DatabaseObjectDecorator;
use wcf\system\application\ApplicationHandler;
use wcf\system\language\LanguageFactory;
use wcf\system\Regex;
use wcf\system\SingletonFactory;
use wcf\system\WCF;
use wcf\util\StringUtil;

/**
 * Handles relative links within the wcf.
 * 
 * @author	<NAME>
 * @copyright	2001-2017 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	WoltLabSuite\Core\System\Request
 */
class LinkHandler extends SingletonFactory {
	/**
	 * regex object to filter title
	 * @var	RegEx
	 */
	protected $titleRegex;
	
	/**
	 * title search strings
	 * @var	string[]
	 */
	protected $titleSearch = [];
	
	/**
	 * title replacement strings
	 * @var	string[]
	 */
	protected $titleReplace = [];
	
	
	/**
	 * @inheritDoc
	 */
	protected function init() {
		$this->titleRegex = new Regex('[\x0-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+');
		
		if (defined('URL_TITLE_COMPONENT_REPLACEMENT') && URL_TITLE_COMPONENT_REPLACEMENT) {
			$replacements = explode("\n", StringUtil::unifyNewlines(StringUtil::trim(URL_TITLE_COMPONENT_REPLACEMENT)));
			foreach ($replacements as $replacement) {
				if (strpos($replacement, '=') === false) continue;
				$components = explode('=', $replacement);
				$this->titleSearch[] = $components[0];
				$this->titleReplace[] = $components[1];
			}
		}
	}
	<commit_msg>Use unicode character properties for URL slugs<commit_after><?php
namespace wcf\system\request;
use wcf\data\page\PageCache;
use wcf\data\DatabaseObjectDecorator;
use wcf\system\application\ApplicationHandler;
use wcf\system\language\LanguageFactory;
use wcf\system\Regex;
use wcf\system\SingletonFactory;
use wcf\system\WCF;
use wcf\util\StringUtil;

/**
 * Handles relative links within the wcf.
 * 
 * @author	<NAME>
 * @copyright	2001-2017 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	WoltLabSuite\Core\System\Request
 */
class LinkHandler extends SingletonFactory {
	/**
	 * regex object to filter title
	 * @var	RegEx
	 */
	protected $titleRegex;
	
	/**
	 * title search strings
	 * @var	string[]
	 */
	protected $titleSearch = [];
	
	/**
	 * title replacement strings
	 * @var	string[]
	 */
	protected $titleReplace = [];
	
	
	/**
	 * @inheritDoc
	 */
	protected function init() {
		$this->titleRegex = new Regex('[^\p{L}\p{N}]+', Regex::UTF_8);
		
		if (defined('URL_TITLE_COMPONENT_REPLACEMENT') && URL_TITLE_COMPONENT_REPLACEMENT) {
			$replacements = explode("\n", StringUtil::unifyNewlines(StringUtil::trim(URL_TITLE_COMPONENT_REPLACEMENT)));
			foreach ($replacements as $replacement) {
				if (strpos($replacement, '=') === false) continue;
				$components = explode('=', $replacement);
				$this->titleSearch[] = $components[0];
				$this->titleReplace[] = $components[1];
			}
		}
	}
	
{'fraction_non_alphanumeric': 0.14875469123166155, 'fraction_numerical': 0.010576595018764927, 'mean_word_length': 4.273381294964029, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5999709', 'n_tokens_mistral': 1093, 'n_tokens_neox': 976, 'n_words': 197}