<commit_before><?
/*
 * Name:				Code blocks
 * Version:		 0.1
 * Plugin URI:	http://trac.hunch.se/rasmus/wiki/wp-plugins#SyntaxHighlight
 * Author:			<NAME>
 * Author URI:	http://hunch.se/
 * Description: Enables defining code blocks using Trac-style {{{...}}} form.
 *							Also highlights code using Pygments, if available.
 * 
 * CSS colors:
 *   You can find a bunch of stylesheets here: 
 */

function code_blocks_dummy_block($content) {
	return '<div><pre>'.h($content).'</pre></div>';
}

function code_blocks_highlight($content, $lang, $cache_ttl=30) {
	global $code_blocks_previous_failure;
	static $pat = 'pygmentize -l %s -f html -P cssclass=';
	
	if (isset($code_blocks_previous_failure) && $code_blocks_previous_failure === true)
		return code_blocks_dummy_block($content);
	
	$cmd = sprintf($pat, escapeshellarg($lang));
	$st = gb::shell($cmd, $content);
	if ($st === null || ($st[0] !== 0 && strpos($st[2], 'command not found') !== false)) {
		# probably no pygments installed -- remember failure in order to speed up multiple calls
		$code_blocks_previous_failure = true;
		gb::log(LOG_WARNING,
			'code-blocks plugin can not highlight code because it can not find pygmentize');
		return code_blocks_dummy_block($content);
	}
	# $st => array(int status, string out, string err)
	if ($st[0] !== 0) {
		gb::log(LOG_WARNING, 'code-blocks plugin failed to highlight code: '.$st[2]);
		return code_blocks_dummy_block($content);
	}
	return $st[1];
}

function code_blocks_wrap($content='') {
	$start = 0;
	while(1) {
		if( ($start = @strpos($content, ($start == 0 ? '{{{' : "\n{{{"), $start)) !== false) {
			if( ($end = strpos($content, "}}}", $start+5)) !== false) {
				$code = trim(substr($content, $start+4, $end-($start+4)));
				$lang = null;
				if( substr($code, 0, 2) == '#!' ) {
					$nl = strpos($code, "\n", 2);
					$lang = trim(substr($code, 2, $nl-2));
					$code = code_blocks_highlight(ltrim(substr($code, $nl+1), "\r"), $lang);
				}
				else {
					$code = code_blocks_dummy_block($code);
				}
				$code_wrapped = base64_encode($code);
				$code_wrapped = sprintf("{{{!%010d%s",strlen($code_wrapped), $code_wrapped);
				$content = substr($content, 0, $start)
					. '<p class="code-block">'.$code_wrapped.'</p>'
					. substr($content, $end+3);
				$start = $end;
			}
			else {
				break;
			}
		}
		else {
			break;
		}
	}
	return $content;
}

function code_blocks_unwrap($content='') {
	$start = 0;
	$content_len = strlen($content);
	while(1) {
		if( ($start = strpos($content, "{{{!", $start)) !== false) {
			$len = intval(substr($content, $start+4, 10));
			$content = substr($content, 0, $start)
				. base64_decode(substr($content, $start+14, $len))
				. substr($content, $start+14+$len);
			$start = $start + 14;
			if($start > $content_len)
				break;
		}
		else {
			break;
		}
	}
	return $content;
}


function code_blocks_init($context) {
	if ($context !== 'rebuild')
		return false;
	GBFilter::add('body.html', 'code_blocks_wrap', 0);
	GBFilter::add('body.html', 'code_blocks_unwrap', 9999);
	return true;
}

?>
<commit_msg>new version of the code block plugin<commit_after><?
/*
 * Name:        Code blocks
 * Version:     0.1
 * Author:      <NAME>
 * Author URI:  http://hunch.se/
 * Description: Enables syntax highlight of <code> blocks using Pygments, if
 *              available.
 */

function code_blocks_config() {
	return array(
		'classname' => '',
		'tabsize' => 2
	);
}

function code_blocks_dummy_block($content) {
	return '<div><pre>'.h($content).'</pre></div>';
}

function code_blocks_highlight($content, $lang, $conf, $input_encoding='utf-8') {
	global $code_blocks_previous_failure;
	
	if (isset($code_blocks_previous_failure) && $code_blocks_previous_failure === true)
		return code_blocks_dummy_block($content);
	
	$cmd = 'pygmentize '.($lang ? '-l '.escapeshellarg($lang) : '-g')
		.' -f html -O cssclass=,encoding='.$input_encoding
		.($conf['tabsize'] ? ',tabsize='.$conf['tabsize'] : '');
	$st = gb::shell($cmd, $content);
	if ($st === null || ($st[0] !== 0 && strpos($st[2], 'command not found') !== false)) {
		# probably no pygments installed -- remember failure in order to speed up multiple calls
		$code_blocks_previous_failure = true;
		gb::log(LOG_WARNING,
			'code-blocks plugin can not highlight code because it can not find pygmentize');
		return code_blocks_dummy_block($content);
	}
	# $st => array(int status, string out, string err)
	if ($st[0] !== 0) {
		gb::log(LOG_WARNING, 'code-blocks plugin failed to highlight code: '.$st[2]);
		return code_blocks_dummy_block($content);
	}
	return $st[1];
}

function code_blocks_filter($text='') {
	$conf = code_blocks_config();
	$tokens = gb_tokenize_html($text);
	$out = '';
	$depth = 0;
	$block = '';
	$tag = '';

	foreach ($tokens as $token) {
		if (substr($token,0,5) === '<code') {
			$depth++;
			if ($depth === 1) {
				# code block just started
				$tag = $token;
				continue;
			}
		}
		elseif (substr($token,0,7) === '</code>') {
			$depth--;
			if ($depth < 0) {
				gb::log(LOG_WARNING, 'stray </code> messing up a code block');
				$depth = 0;
			}
			if ($depth === 0) {
				# code block ended
				if ($block) {
					$lang = '';
					# find lang, if any
					if (preg_match('/[\s\t ]+lang=("[^"]*"|\'[^\']*\')[\s\t ]*/', $tag, $m, PREG_OFFSET_CAPTURE)) {
						$lang = trim($m[1][0], '"\'');
						$end = substr($tag, $m[0][1]+strlen($m[0][0]));
						$tag = substr($tag, 0, $m[0][1]).($end === '>' ? '>' : ' '.$end);
					}
					# add CSS class name
					if ($conf['classname']) {
						if (($p = strpos($tag, 'class=')) !== false)
							$tag = substr($tag, 0, $p+7) . $conf['classname'].' ' . substr($tag, $p+7);
						else
							$tag = substr($tag, 0, -1) . ' class="'.$conf['classname'].'">';
					}
					# remove first and last line break if present
					if ($block{0} === "\n")
						$block = substr($block, 1);
					if ($block{strlen($block)-1} === "\n")
						$block = substr($block, 0, -1);
					# expand tabs
					if ($conf['tabsize'])
						$block = strtr($block, array("\t" => str_repeat(' ', $conf['tabsize'])));
					# append block to output
					$out .= $tag . code_blocks_highlight($block, $lang, $conf) . '</code>';
					# clear block
					$block = '';
				}
				continue;
			}
		}

		# in codeblock or not?
		if ($depth)
			$block .= $token;
		else
			$out .= $token;
	}
	
	return $out;
}


function code_blocks_init($context) {
	if ($context !== 'rebuild')
		return false;
	GBFilter::add('body.html', 'code_blocks_filter', 0);
	return true;
}

?>

{'fraction_non_alphanumeric': 0.20552570990023022, 'fraction_numerical': 0.013660782808902533, 'mean_word_length': 3.5662228451296425, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4590518', 'n_tokens_mistral': 2701, 'n_tokens_neox': 2338, 'n_words': 544}