<commit_before><?php
namespace wcf\system\search;
use wcf\system\bbcode\KeywordHighlighter;
use wcf\system\Regex;
use wcf\system\SingletonFactory;
use wcf\util\ArrayUtil;
use wcf\util\StringUtil;

/**
 * Formats messages for search result output.
 * 
 * @author	<NAME>
 * @copyright	2001-2016 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	com.woltlab.wcf
 * @subpackage	system.search
 * @category	Community Framework
 */
class SearchResultTextParser extends SingletonFactory {
	/**
	 * max length for message abstract
	 * @var	integer
	 */
	const MAX_LENGTH = 500;
	
	/**
	 * highlight query
	 * @var	mixed
	 */
	protected $searchQuery = '';
	
	/**
	 * @inheritDoc
	 */
	protected function init() {
		if (isset($_GET['highlight'])) {
			$keywordString = $_GET['highlight'];
			
			// remove search operators
			$keywordString = preg_replace('/[\+\-><()~\*]+/', '', $keywordString);
			
			if (mb_substr($keywordString, 0, 1) == '"' && mb_substr($keywordString, -1) == '"') {
				// phrases search
				$keywordString = StringUtil::trim(mb_substr($keywordString, 1, -1));
				
				if (!empty($keywordString)) {
					$this->searchQuery = $keywordString;
				}
			}
			else {
				$this->searchQuery = ArrayUtil::trim(explode(' ', $keywordString));
				if (empty($this->searchQuery)) {
					$this->searchQuery = false;
				}
				else if (count($this->searchQuery) == 1) {
					$this->searchQuery = reset($this->searchQuery);
				}
			}
		}
	}
	
	/**
	 * Returns an abstract of the given message.
	 * Uses search keywords to shift the start and end position of the abstract (like Google).
	 * 
	 * @param	string		$text
	 * @return	string
	 */
	protected function getMessageAbstract($text) {
		// replace newlines with spaces
		$text = Regex::compile("\s+")->replace($text, ' ');
		
		if (mb_strlen($text) > static::MAX_LENGTH) {
			if ($this->searchQuery) {
				// phrase search
				if (!is_array($this->searchQuery)) {
					$start = mb_strripos($text, $this->searchQuery);
					if ($start !== false) {
						$end = $start + mb_strlen($this->searchQuery);
						$shiftStartBy = $shiftEndBy = round((static::MAX_LENGTH - mb_strlen($this->searchQuery)) / 2);
						
						// shiftStartBy is negative when search query length is over max length
						if ($shiftStartBy < 0) {
							$shiftEndBy += $shiftStartBy;
							$shiftStartBy = 0;
						}
						
						// shift abstract start
						if ($start - $shiftStartBy < 0) {
							$shiftEndBy += $shiftStartBy - $start;
							$start = 0;
						}
						else {
							$start -= $shiftStartBy;
						}
						
						// shift abstract end
						if ($end + $shiftEndBy > mb_strlen($text) - 1) {
							$shiftStartBy = $end + $shiftEndBy - mb_strlen($text) - 1;
							if ($shiftStartBy > $start) {
								$start = 0;
							}<commit_msg>Fixed UTF-8 issue in text parser<commit_after><?php
namespace wcf\system\search;
use wcf\system\bbcode\KeywordHighlighter;
use wcf\system\Regex;
use wcf\system\SingletonFactory;
use wcf\util\ArrayUtil;
use wcf\util\StringUtil;

/**
 * Formats messages for search result output.
 * 
 * @author	<NAME>
 * @copyright	2001-2016 WoltLab GmbH
 * @license	GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
 * @package	com.woltlab.wcf
 * @subpackage	system.search
 * @category	Community Framework
 */
class SearchResultTextParser extends SingletonFactory {
	/**
	 * max length for message abstract
	 * @var	integer
	 */
	const MAX_LENGTH = 500;
	
	/**
	 * highlight query
	 * @var	mixed
	 */
	protected $searchQuery = '';
	
	/**
	 * @inheritDoc
	 */
	protected function init() {
		if (isset($_GET['highlight'])) {
			$keywordString = $_GET['highlight'];
			
			// remove search operators
			$keywordString = preg_replace('/[\+\-><()~\*]+/', '', $keywordString);
			
			if (mb_substr($keywordString, 0, 1) == '"' && mb_substr($keywordString, -1) == '"') {
				// phrases search
				$keywordString = StringUtil::trim(mb_substr($keywordString, 1, -1));
				
				if (!empty($keywordString)) {
					$this->searchQuery = $keywordString;
				}
			}
			else {
				$this->searchQuery = ArrayUtil::trim(explode(' ', $keywordString));
				if (empty($this->searchQuery)) {
					$this->searchQuery = false;
				}
				else if (count($this->searchQuery) == 1) {
					$this->searchQuery = reset($this->searchQuery);
				}
			}
		}
	}
	
	/**
	 * Returns an abstract of the given message.
	 * Uses search keywords to shift the start and end position of the abstract (like Google).
	 * 
	 * @param	string		$text
	 * @return	string
	 */
	protected function getMessageAbstract($text) {
		// replace newlines with spaces
		$text = Regex::compile("\s+", Regex::UTF_8)->replace($text, ' ');
		
		if (mb_strlen($text) > static::MAX_LENGTH) {
			if ($this->searchQuery) {
				// phrase search
				if (!is_array($this->searchQuery)) {
					$start = mb_strripos($text, $this->searchQuery);
					if ($start !== false) {
						$end = $start + mb_strlen($this->searchQuery);
						$shiftStartBy = $shiftEndBy = round((static::MAX_LENGTH - mb_strlen($this->searchQuery)) / 2);
						
						// shiftStartBy is negative when search query length is over max length
						if ($shiftStartBy < 0) {
							$shiftEndBy += $shiftStartBy;
							$shiftStartBy = 0;
						}
						
						// shift abstract start
						if ($start - $shiftStartBy < 0) {
							$shiftEndBy += $shiftStartBy - $start;
							$start = 0;
						}
						else {
							$start -= $shiftStartBy;
						}
						
						// shift abstract end
						if ($end + $shiftEndBy > mb_strlen($text) - 1) {
							$shiftStartBy = $end + $shiftEndBy - mb_strlen($text) - 1;
							if ($shiftStartBy > $start) {
								$start = 0;
							}
{'fraction_non_alphanumeric': 0.15459780143081486, 'fraction_numerical': 0.009073460129122317, 'mean_word_length': 2.8677462887989202, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6348133', 'n_tokens_mistral': 2335, 'n_tokens_neox': 1802, 'n_words': 432}