<commit_before><?php
// Northpole.fi
// Tekstinkäsittely
// 16.2.2010

function bbcode_format($str) {

		$str = str_replace("javascript:", "", $str);

		$simple_search = array(
				'/\[b\](.*?)\[\/b\]/is',
				'/\[i\](.*?)\[\/i\]/is',
				'/\[u\](.*?)\[\/u\]/is',
				'/\[s\](.*?)\[\/s\]/is',
				'/\[align\=(left|center|right)\](.*?)\[\/align\]/is',
				'/\[mail\=(.*?)\](.*?)\[\/mail\]/is',
				'/\[mail\](.*?)\[\/mail\]/is',
				'/\[url\=www\.(.*?)\](.*?)\[\/url\]/is',
				'/\[url\]www\.(.*?)\[\/url\]/is',
				'/\[url\=(.*?)\](.*?)\[\/url\]/is',
				'/\[url\](.*?)\[\/url\]/is',
				'/\[spoiler\](.*?)\[\/spoiler\]/is',
				'/\[code\](.*?)\[\/code\]/is',
				'/\[quote\]([^\[]+)\[\/quote\]/is',
				'/\[quote\=([^\]]+)\]([^\[]+)\[\/quote\]/is'
				);
				//'/\[color\=(.*?)\](.*?)\[\/color\]/is',

		$simple_replace = array(
				'<strong>$1</strong>',
				'<em>$1</em>',
				'<u>$1</u>',
				'<span style="text-decoration:line-through;">$1</span>',
				'</p><div style="text-align: $1;"><p>$2</p></div><p>',
				'<a href="mailto:$1">$2</a>',
				'<a href="mailto:$1">$1</a>',
				'<a href="http://www.$1">$2</a>',
				'<a href="http://www.$1">$1</a>',
				'<a href="$1">$2</a>',
				'<a href="$1">$1</a>',
				'<span class="spoiler">$1</span>',
				'</p><pre>$1</pre><p>',
				'</p><div class="bbquote"><p>$1</p></div><p>',
				'</p><div class="bbquote"><p>$2<span>&mdash; $1</span></p></div><p>'
				);
				//'<span style="color: $1;">$2</span>',
				
		// Do simple BBCode's
		$str = preg_replace($simple_search, $simple_replace, $str);

		return $str;
}

// Thanks goes to Koukari!
function regulateWords( $data, $max_length = 120 )
{
	if( $max_length <= 0 )
		return $data;

	$length = strlen( $data );
	$pos = 0;			// Current position
	$tag = false;		// Are we inside a tag atm?
	$wordLength = 0;	// The length of the current word

	while( $pos < $length )
	{
		if( $tag == true )
		{
			if( $data[$pos] == '>' )
				$tag = false;
		}

		else
		{
			switch( $data[$pos] )
			{
				case '<':
					$tag = true;
					break;

				case ' ':
					$wordLength = 0;
					break;

				case "\n":
					$wordLength = 0;
					break;

				default:
					$wordLength++;
					break;
			}

			if( $wordLength > $max_length )
			{
				$sub1 = substr( $data, 0, $pos );
				$sub2 = substr( $data, $pos );
				$data = $sub1.' '.$sub2;
				$wordLength = 0;
				$length++;
			}
		}
		$pos++;
	}
	return $data;
}

function removeForbiddenUnicode($text)
{
	// Remove invisible characters and characters that mess up the formatting.
	$unicode = array(
		'/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/',	// Unicode control characters
		'/'. pack("cc", 0xC2, 0xAD) .'/',			// 'SOFT HYPHEN' (U+00AD)
		'/'. pack("ccc", 0xE1, 0x85, 0x9F) .'/',	// 'HANGUL CHOSEONG FILLER' (U+115F)
		'/'. pack("ccc", 0xE2, 0x80, 0x8B) .'/',	// 'ZERO WIDTH SPACE' (U+200B)
		'/'. pack("ccc", 0xE2, 0x80, 0x8C) .'/',	// 'ZERO WIDTH NON-JOINER' (U+200C)
		'/'. pack("ccc", 0xE2, 0x80, 0x8D) .'/',	// 'ZERO WIDTH JOINER' (U+200D)
		'/'. pack("ccc", 0xE2, 0x80, 0x8E) .'/',	// 'LEFT-TO-RIGHT MARK' (U+200E)
		'/'. pack("ccc", 0xE2, 0x80, 0x8F) .'/',	// 'RIGHT-TO-LEFT MARK' (U+200F)
		'/'. pack("ccc", 0xE2, 0x80, 0xA8) .'/',	// 'LINE SEPARATOR' (U+2028)
		'/'. pack("ccc", 0xE2, 0x80, 0xA9) .'/',	// 'PARAGRAPH SEPARATOR' (U+2029)
		'/'. pack("ccc", 0xE2, 0x80, 0xAA) .'/',	// 'LEFT-TO-RIGHT EMBEDDING' (U+202A)
		'/'. pack("ccc", 0xE2, 0x80, 0xAB) .'/',	// 'RIGHT-TO-LEFT EMBEDDING' (U+202B)
		'/'. pack("ccc", 0xE2, 0x80, 0xAE) .'/',	// 'RIGHT-TO-LEFT OVERRIDE' (U+202E)
		'/'. pack("ccc", 0xE2, 0x81, 0xA0) .'/',	// 'WORD JOINER' (U+2060)
		'/'. pack("ccc", 0xE2, 0x81, 0xA1) .'/',	// 'FUNCTION APPLICATION' (U+2061)
		'/'. pack("ccc", 0xE2, 0x81, 0xA2) .'/',	// 'INVISIBLE TIMES' (U+2062)
		'/'. pack("ccc", 0xE2, 0x81, 0xA3) .'/',	// 'INVISIBLE SEPARATOR' (U+2063)
		'/'. pack("ccc", 0xE2, 0x81, 0xA4) .'/',	// 'INVISIBLE PLUS' (U+2064)
		'/'. pack("ccc", 0xEF, 0xB8, 0x80) .'/',	// 'VARIATION SELECTOR-1' (U+FE00)
		'/'. pack("ccc", 0xEF, 0xB8, 0x81) .'/',	// 'VARIATION SELECTOR-2' (U+FE01)
		'/'. pack("ccc", 0xEF, 0xB8, 0x82) .'/',	// 'VARIATION SELECTOR-3' (U+FE02)
		'/'. pack("ccc", 0xEF, 0xB8, 0x83) .'/',	// 'VARIATION SELECTOR-4' (U+FE03)
		'/'. pack("ccc", 0xEF, 0xB8, 0x84) .'/',	// 'VARIATION SELECTOR-5' (U+FE04)
		'/'. pack("ccc", 0xEF, 0xB8, 0x85) .'/',	// 'VARIATION SELECTOR-6' (U+FE05)
		'/'. pack("ccc", 0xEF, 0xB8, 0x86) .'/',	// 'VARIATION SELECTOR-7' (U+FE06)
		'/'. pack("ccc", 0xEF, 0xB8, 0x87) .'/',	// 'VARIATION SELECTOR-8' (U+FE07)
		'/'. pack("ccc", 0xEF, 0xB8, 0x88) .'/',	// 'VARIATION SELECTOR-9' (U+FE08)
		'/'. pack("ccc", 0xEF, 0xB8, 0x89) .'/',	// 'VARIATION SELECTOR-10' (U+FE09)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8A) .'/',	// 'VARIATION SELECTOR-11' (U+FE0A)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8B) .'/',	// 'VARIATION SELECTOR-12' (U+FE0B)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8C) .'/',	// 'VARIATION SELECTOR-13' (U+FE0C)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8D) .'/',	// 'VARIATION SELECTOR-14' (U+FE0D)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8E) .'/',	// 'VARIATION SELECTOR-15' (U+FE0E)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8F) .'/',	// 'VARIATION SELECTOR-16' (U+FE0F)
		'/'. pack("ccc", 0xEF, 0xBB, 0xBF) .'/',	// 'ZERO WIDTH NO-BREAK SPACE' (U+FEFF)
	);
	$text = preg_replace($unicode, '', $text);
	
	return $text;
}

// Tekstin muotoilua
function format_text($text) {
	global $cfg;
	
	// Karsitaan kautta... keno.. mitkälie, ja hienosäädetään
	// $text = str_replace('&amp;', '&', htmlspecialchars($text));
	$text = htmlspecialchars($text);
	
	// Remove unicode control characters
	$text = removeForbiddenUnicode($text);
	
	$text = preg_replace("#(&gt;&gt;)([0-9]+)#ise", "'<a href=\"http://". $cfg['htmldir_plain'] ."/redirect/\\2\" title=\"ajax;http://". $cfg['htmldir_plain'] ."/scripts/ajax/message.php?id=\\2\" onclick=\"highlight_post(\'\\2\', this); return false;\">\\1\\2</a>'", $text);
	$text = preg_replace("#(^|[\n\]])(&gt;)([^\n\r]+)#ise", "'\\1<span class=\"quote\">\\2\\3</span>'", $text);	
	
	// Ja tähän BBkoodisysteemi..
	$text = bbcode_format($text);
	
	// Automaattinen korvaus ***://, www., mailto:, ftp. teksteille linkeiksi!
        // Needs more testing, does not work.
	//$text = preg_replace("#\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/)))#iS", "<a href=\"\\1\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\1</a>", $text);
	$text = preg_replace("#(^|[\n \>\(])([\w]+?://[\w]+[^\) \"\n\r\t<]*)#ise", "'\\1<a href=\"\\2\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\2</a>'", $text);
	$text = preg_replace("#(^|[\n \>\(])((www)\.[^\) \"\t\n\r<]*)#ise", "'\\1<a href=\"http://\\2\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\2</a>'", $text);
	$text = preg_replace("#(^|[\n \>\(])((ftp)\.[^\) \"\t\n\r<]*)#ise", "'\\1<a href=\"ftp://\\2\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\2</a>'", $text);
	$text = preg_replace("#(^|[\n \>\(])([a-z0-9&\-_\.]+?)@([\w\-]+\.([\w\-\.]+\.)*[\)\w]+)#i", "\\1<a href=\"mailto:\\2@\\3\">\\2@\\3</a>", $text);

	// Karsitaan liiat rivinvaihdot
	$text = preg_replace('/(\n|(\r\n)){11,}/', "\n\n\n\n\n\n\n\n\n\n", $text);
	
	$text = trim($text);
	$text = regulateWords($text, $cfg['max_word_length']);
	
	// Ja viimeisenä vielä poistetaan MySQL-koodi tekstistä
	$text = mysql_real_escape_string($text);

	return $text;
}

// Tekstin lyhentäminen
function truncate($text, $messageid, $length = 128, $link = false, $truncated_text = true) {

	if(!empty($text)) {
		
		if($link) {
			$a = '<a href="'. $link .'" onclick="unTruncate('. $messageid .'); return false;">';
			$b = '</a>';
			$c = T_("here");
		}<commit_msg>fixed issue on commin bab52f29a79288f4f9db5e3bb85a614f075b497e (propably)<commit_after><?php
// Northpole.fi
// Tekstinkäsittely
// 16.2.2010

function bbcode_format($str) {

		$str = str_replace("javascript:", "", $str);

		$simple_search = array(
				'/\[b\](.*?)\[\/b\]/is',
				'/\[i\](.*?)\[\/i\]/is',
				'/\[u\](.*?)\[\/u\]/is',
				'/\[s\](.*?)\[\/s\]/is',
				'/\[align\=(left|center|right)\](.*?)\[\/align\]/is',
				'/\[mail\=(.*?)\](.*?)\[\/mail\]/is',
				'/\[mail\](.*?)\[\/mail\]/is',
				'/\[url\=www\.(.*?)\](.*?)\[\/url\]/is',
				'/\[url\]www\.(.*?)\[\/url\]/is',
				'/\[url\=(.*?)\](.*?)\[\/url\]/is',
				'/\[url\](.*?)\[\/url\]/is',
				'/\[spoiler\](.*?)\[\/spoiler\]/is',
				'/\[code\](.*?)\[\/code\]/is',
				'/\[quote\]([^\[]+)\[\/quote\]/is',
				'/\[quote\=([^\]]+)\]([^\[]+)\[\/quote\]/is'
				);
				//'/\[color\=(.*?)\](.*?)\[\/color\]/is',

		$simple_replace = array(
				'<strong>$1</strong>',
				'<em>$1</em>',
				'<u>$1</u>',
				'<span style="text-decoration:line-through;">$1</span>',
				'</p><div style="text-align: $1;"><p>$2</p></div><p>',
				'<a href="mailto:$1">$2</a>',
				'<a href="mailto:$1">$1</a>',
				'<a href="http://www.$1">$2</a>',
				'<a href="http://www.$1">$1</a>',
				'<a href="$1">$2</a>',
				'<a href="$1">$1</a>',
				'<span class="spoiler">$1</span>',
				'</p><pre>$1</pre><p>',
				'</p><div class="bbquote"><p>$1</p></div><p>',
				'</p><div class="bbquote"><p>$2<span>&mdash; $1</span></p></div><p>'
				);
				//'<span style="color: $1;">$2</span>',
				
		// Do simple BBCode's
		$str = preg_replace($simple_search, $simple_replace, $str);

		return $str;
}

// Thanks goes to Koukari!
function regulateWords( $data, $max_length = 120 )
{
	if( $max_length <= 0 )
		return $data;

	$length = strlen( $data );
	$pos = 0;			// Current position
	$tag = false;		// Are we inside a tag atm?
	$wordLength = 0;	// The length of the current word

	while( $pos < $length )
	{
		if( $tag == true )
		{
			if( $data[$pos] == '>' )
				$tag = false;
		}

		else
		{
			switch( $data[$pos] )
			{
				case '<':
					$tag = true;
					break;

				case ' ':
					$wordLength = 0;
					break;

				case "\n":
					$wordLength = 0;
					break;

				default:
					$wordLength++;
					break;
			}

			if( $wordLength > $max_length )
			{
				$sub1 = substr( $data, 0, $pos );
				$sub2 = substr( $data, $pos );
				$data = $sub1.' '.$sub2;
				$wordLength = 0;
				$length++;
			}
		}
		$pos++;
	}
	return $data;
}

function removeForbiddenUnicode($text)
{
	// Remove invisible characters and characters that mess up the formatting.
	$unicode = array(
		'/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/',	// Unicode control characters
		'/'. pack("cc", 0xC2, 0xAD) .'/',			// 'SOFT HYPHEN' (U+00AD)
		'/'. pack("ccc", 0xE1, 0x85, 0x9F) .'/',	// 'HANGUL CHOSEONG FILLER' (U+115F)
		'/'. pack("ccc", 0xE2, 0x80, 0x8B) .'/',	// 'ZERO WIDTH SPACE' (U+200B)
		'/'. pack("ccc", 0xE2, 0x80, 0x8C) .'/',	// 'ZERO WIDTH NON-JOINER' (U+200C)
		'/'. pack("ccc", 0xE2, 0x80, 0x8D) .'/',	// 'ZERO WIDTH JOINER' (U+200D)
		'/'. pack("ccc", 0xE2, 0x80, 0x8E) .'/',	// 'LEFT-TO-RIGHT MARK' (U+200E)
		'/'. pack("ccc", 0xE2, 0x80, 0x8F) .'/',	// 'RIGHT-TO-LEFT MARK' (U+200F)
		'/'. pack("ccc", 0xE2, 0x80, 0xA8) .'/',	// 'LINE SEPARATOR' (U+2028)
		'/'. pack("ccc", 0xE2, 0x80, 0xA9) .'/',	// 'PARAGRAPH SEPARATOR' (U+2029)
		'/'. pack("ccc", 0xE2, 0x80, 0xAA) .'/',	// 'LEFT-TO-RIGHT EMBEDDING' (U+202A)
		'/'. pack("ccc", 0xE2, 0x80, 0xAB) .'/',	// 'RIGHT-TO-LEFT EMBEDDING' (U+202B)
		'/'. pack("ccc", 0xE2, 0x80, 0xAE) .'/',	// 'RIGHT-TO-LEFT OVERRIDE' (U+202E)
		'/'. pack("ccc", 0xE2, 0x81, 0xA0) .'/',	// 'WORD JOINER' (U+2060)
		'/'. pack("ccc", 0xE2, 0x81, 0xA1) .'/',	// 'FUNCTION APPLICATION' (U+2061)
		'/'. pack("ccc", 0xE2, 0x81, 0xA2) .'/',	// 'INVISIBLE TIMES' (U+2062)
		'/'. pack("ccc", 0xE2, 0x81, 0xA3) .'/',	// 'INVISIBLE SEPARATOR' (U+2063)
		'/'. pack("ccc", 0xE2, 0x81, 0xA4) .'/',	// 'INVISIBLE PLUS' (U+2064)
		'/'. pack("ccc", 0xEF, 0xB8, 0x80) .'/',	// 'VARIATION SELECTOR-1' (U+FE00)
		'/'. pack("ccc", 0xEF, 0xB8, 0x81) .'/',	// 'VARIATION SELECTOR-2' (U+FE01)
		'/'. pack("ccc", 0xEF, 0xB8, 0x82) .'/',	// 'VARIATION SELECTOR-3' (U+FE02)
		'/'. pack("ccc", 0xEF, 0xB8, 0x83) .'/',	// 'VARIATION SELECTOR-4' (U+FE03)
		'/'. pack("ccc", 0xEF, 0xB8, 0x84) .'/',	// 'VARIATION SELECTOR-5' (U+FE04)
		'/'. pack("ccc", 0xEF, 0xB8, 0x85) .'/',	// 'VARIATION SELECTOR-6' (U+FE05)
		'/'. pack("ccc", 0xEF, 0xB8, 0x86) .'/',	// 'VARIATION SELECTOR-7' (U+FE06)
		'/'. pack("ccc", 0xEF, 0xB8, 0x87) .'/',	// 'VARIATION SELECTOR-8' (U+FE07)
		'/'. pack("ccc", 0xEF, 0xB8, 0x88) .'/',	// 'VARIATION SELECTOR-9' (U+FE08)
		'/'. pack("ccc", 0xEF, 0xB8, 0x89) .'/',	// 'VARIATION SELECTOR-10' (U+FE09)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8A) .'/',	// 'VARIATION SELECTOR-11' (U+FE0A)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8B) .'/',	// 'VARIATION SELECTOR-12' (U+FE0B)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8C) .'/',	// 'VARIATION SELECTOR-13' (U+FE0C)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8D) .'/',	// 'VARIATION SELECTOR-14' (U+FE0D)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8E) .'/',	// 'VARIATION SELECTOR-15' (U+FE0E)
		'/'. pack("ccc", 0xEF, 0xB8, 0x8F) .'/',	// 'VARIATION SELECTOR-16' (U+FE0F)
		'/'. pack("ccc", 0xEF, 0xBB, 0xBF) .'/',	// 'ZERO WIDTH NO-BREAK SPACE' (U+FEFF)
	);
	$text = preg_replace($unicode, '', $text);
	
	return $text;
}

// Tekstin muotoilua
function format_text($text) {
	global $cfg;
	
	// Karsitaan kautta... keno.. mitkälie, ja hienosäädetään
	// $text = str_replace('&amp;', '&', htmlspecialchars($text));
	$text = htmlspecialchars($text);
	
	// Remove unicode control characters
	$text = removeForbiddenUnicode($text);
	
        // Automaattinen korvaus ***://, www., mailto:, ftp. teksteille linkeiksi! V2!
        $text = preg_replace("/(([\da-z\.-]+):\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([#\?%\~\;\=&\/\w \.-]*)*\/?/", "<a href=\"\\0\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\0</a>", $text);
     	//$text = preg_replace("#(^|[\n \>\(])([\w]+?://[\w]+[^\) \"\n\r\t<]*)#ise", "'\\1<a href=\"\\2\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\2</a>'", $text);
	//$text = preg_replace("#(^|[\n \>\(])((www)\.[^\) \"\t\n\r<]*)#ise", "'\\1<a href=\"http://\\2\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\2</a>'", $text);
	//$text = preg_replace("#(^|[\n \>\(])((ftp)\.[^\) \"\t\n\r<]*)#ise", "'\\1<a href=\"ftp://\\2\" rel=\"nofollow\" onclick=\"window.open(this.href); return false;\">\\2</a>'", $text);
	//$text = preg_replace("#(^|[\n \>\(])([a-z0-9&\-_\.]+?)@([\w\-]+\.([\w\-\.]+\.)*[\)\w]+)#i", "\\1<a href=\"mailto:\\2@\\3\">\\2@\\3</a>", $text);
        
        // viestilinkit ja vihertekstit
	$text = preg_replace("#(&gt;&gt;)([0-9]+)#ise", "'<a href=\"http://". $cfg['htmldir_plain'] ."/redirect/\\2\" title=\"ajax;http://". $cfg['htmldir_plain'] ."/scripts/ajax/message.php?id=\\2\" onclick=\"highlight_post(\'\\2\', this); return false;\">\\1\\2</a>'", $text);
	$text = preg_replace("#(^|[\n\]])(&gt;)([^\n\r]+)#ise", "'\\1<span class=\"quote\">\\2\\3</span>'", $text);	
	
	// Ja tähän BBkoodisysteemi..
	$text = bbcode_format($text);
	

	// Karsitaan liiat rivinvaihdot
	$text = preg_replace('/(\n|(\r\n)){11,}/', "\n\n\n\n\n\n\n\n\n\n", $text);
	
	$text = trim($text);
	$text = regulateWords($text, $cfg['max_word_length']);
	
	// Ja viimeisenä vielä poistetaan MySQL-koodi tekstistä
	$text = mysql_real_escape_string($text);

	return $text;
}

// Tekstin lyhentäminen
function truncate($text, $messageid, $length = 128, $link = false, $truncated_text = true) {

	if(!empty($text)) {
		
		if($link) {
			$a = '<a href="'. $link .'" onclick="unTruncate('. $messageid .'); return false;">';
			$b = '</a>';
			$c = T_("here");
		}
{'fraction_non_alphanumeric': 0.32411783748786016, 'fraction_numerical': 0.0534153447717708, 'mean_word_length': 2.9183155758498223, 'pattern_counts': {'":': 2, '<': 146, '<?xml version=': 0, '>': 144, 'https://': 0, 'lorem ipsum': 0, 'www.': 13, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5179555', 'n_tokens_mistral': 8357, 'n_tokens_neox': 7000, 'n_words': 1157}