<commit_before><?php
require_once('tiki-setup.php');

// Make sure script is run from a shell
if (PHP_SAPI !== 'cli') {
	die("Please run for a shell");
}


// Show a description of this tool
echo "\nhtmlconvert: test tool for Jison parser\n";
echo "---------------------------------------\n";

// Interpret command line

// Display help if no arguments are passed
if(count($argv) <= 1) {
	$argv[] = '-?';
}

// Help
if (in_array('-?', $argv)) {
	echo "Command line arguments\n";
	echo "  -?	Show this help text\n";
	echo "  -f	Generate a full report\n";
	echo "  -x	Exclude the page contents from the report\n";
	echo "\n";
	die;
}

// Hide content reporting
$xcludeContent = false;
if (in_array('-x', $argv)) {
	echo "Hiding reported content.\n";
	$xcludeContent = true;
}

// Full content reporting
if (in_array('-f', $argv)) {
	echo "Full report.\n";
	$xcludeContent = false;
}


// The test function
///////////////////////////
function tf($page, &$startTime, &$endTime) {
	global $xcludeContent;
	
	//The new parser strips all \r and lets \n do all the line break work
	$syntax = $page['data'];
	$syntax = str_replace("\r", '', $syntax);

	// Determine the page syntax type
	$is_html = isset($page['is_html']) ? $page['is_html'] == 1 : false;

	$parser = new JisonParser_Wiki_Handler();
	$WysiwygParser = new JisonParser_WikiCKEditor_Handler();
	$parserHtmlToWiki = new JisonParser_Html_Handler();

	// Parse
	$startTime = getMicroTime();
	if(!$is_html) {
		$wikiSyntax = $syntax;
		$html = $WysiwygParser->parse($wikiSyntax);
		$wiki = $parserHtmlToWiki->parse($html);
	} else {
		$htmlSyntax = $syntax;
		$wikiSyntax = $parserHtmlToWiki->parse($htmlSyntax);
		$html = $WysiwygParser->parse($wikiSyntax);
		$wiki = $parserHtmlToWiki->parse($html);
	}
	$endTime = getMicroTime();

	$success =  $wikiSyntax == $wiki;
	
	if($success == false && !$xcludeContent) {
		echo "\n";
		echo '"' . $wikiSyntax . '"';
		echo "\n---------------------" . mb_detect_encoding($wikiSyntax) . "-------------------------\n";
		echo $html;
		echo "\n----------------------------------------------\n";
		echo '"' . $wiki . '"';
		echo "\n----------------------" . mb_detect_encoding($wiki) . "------------------------\n";
	}
	echo ($success  ? "\tSUCCESS" : "\tFAILURE");

	unset($parser);
	unset($WysiwygParser);
	unset($parserHtmlToWiki);
	unset($html);
	
	return $success;
}

// Run the process
/////////////////////
global $tikilib;

echo "Loading ALL wiki pages from the database...\n";
$pages = $tikilib->fetchAll("SELECT pageName, data, is_html from tiki_pages");
$pageCount = count($pages);

echo "Analyzing wiki pages...\n";
$cntSUCCESS = 0;
$cntFAILURE = 0;
$totalElapsedTime = 0;
$idx = 0;
foreach($pages as &$page) {
	
	// Processing preparation
	$idx++;
	echo "Processing (".$idx."/".$pageCount.") - ".$page['pageName']." - ";
	
	// Process the page
	if (tf($page, $startTime, $endTime)) {
		++$cntSUCCESS;
	} else {
		++$cntFAILURE;
	}
	unset($page);

	// page_Parser statistics	
	$totalTime = $endTime - $startTime;
	$totalElapsedTime += $totalTime;
	echo " - ".(int)$totalTime." ms\n";

}
echo "\n------------- Statistics ---------------------------------\n";

// Success / Failure
echo "\nSUCCESS: ".$cntSUCCESS."\n";
echo "FAILURE: ".$cntFAILURE."\n";

// Time spent
$avgTimePerPage = "N/A";
if($idx > 0) {
	$avgTimePerPage = $totalElapsedTime / $idx;
}
echo "\nTotal elapsed msec: ".$totalElapsedTime."\n";
echo "Avg msec/page: ".$avgTimePerPage."\n";


// Support functions
///////////////////////

// @return current micro time in milli-seconds
function getMicroTime()
{
	$mtime = microtime(); 
	$mtime = explode(" ",$mtime); 
	$mtime = $mtime[1] + $mtime[0]; 
	return $mtime * 1000;	// Convert to milliseconds
}<commit_msg>[FIX] spelling error<commit_after><?php
require_once('tiki-setup.php');

// Make sure script is run from a shell
if (PHP_SAPI !== 'cli') {
	die("Please run from a shell");
}


// Show a description of this tool
echo "\nhtmlconvert: test tool for Jison parser\n";
echo "---------------------------------------\n";

// Interpret command line

// Display help if no arguments are passed
if(count($argv) <= 1) {
	$argv[] = '-?';
}

// Help
if (in_array('-?', $argv)) {
	echo "Command line arguments\n";
	echo "  -?	Show this help text\n";
	echo "  -f	Generate a full report\n";
	echo "  -x	Exclude the page contents from the report\n";
	echo "\n";
	die;
}

// Hide content reporting
$xcludeContent = false;
if (in_array('-x', $argv)) {
	echo "Hiding reported content.\n";
	$xcludeContent = true;
}

// Full content reporting
if (in_array('-f', $argv)) {
	echo "Full report.\n";
	$xcludeContent = false;
}


// The test function
///////////////////////////
function tf($page, &$startTime, &$endTime) {
	global $xcludeContent;
	
	//The new parser strips all \r and lets \n do all the line break work
	$syntax = $page['data'];
	$syntax = str_replace("\r", '', $syntax);

	// Determine the page syntax type
	$is_html = isset($page['is_html']) ? $page['is_html'] == 1 : false;

	$parser = new JisonParser_Wiki_Handler();
	$WysiwygParser = new JisonParser_WikiCKEditor_Handler();
	$parserHtmlToWiki = new JisonParser_Html_Handler();

	// Parse
	$startTime = getMicroTime();
	if(!$is_html) {
		$wikiSyntax = $syntax;
		$html = $WysiwygParser->parse($wikiSyntax);
		$wiki = $parserHtmlToWiki->parse($html);
	} else {
		$htmlSyntax = $syntax;
		$wikiSyntax = $parserHtmlToWiki->parse($htmlSyntax);
		$html = $WysiwygParser->parse($wikiSyntax);
		$wiki = $parserHtmlToWiki->parse($html);
	}
	$endTime = getMicroTime();

	$success =  $wikiSyntax == $wiki;
	
	if($success == false && !$xcludeContent) {
		echo "\n";
		echo '"' . $wikiSyntax . '"';
		echo "\n---------------------" . mb_detect_encoding($wikiSyntax) . "-------------------------\n";
		echo $html;
		echo "\n----------------------------------------------\n";
		echo '"' . $wiki . '"';
		echo "\n----------------------" . mb_detect_encoding($wiki) . "------------------------\n";
	}
	echo ($success  ? "\tSUCCESS" : "\tFAILURE");

	unset($parser);
	unset($WysiwygParser);
	unset($parserHtmlToWiki);
	unset($html);
	
	return $success;
}

// Run the process
/////////////////////
global $tikilib;

echo "Loading ALL wiki pages from the database...\n";
$pages = $tikilib->fetchAll("SELECT pageName, data, is_html from tiki_pages");
$pageCount = count($pages);

echo "Analyzing wiki pages...\n";
$cntSUCCESS = 0;
$cntFAILURE = 0;
$totalElapsedTime = 0;
$idx = 0;
foreach($pages as &$page) {
	
	// Processing preparation
	$idx++;
	echo "Processing (".$idx."/".$pageCount.") - ".$page['pageName']." - ";
	
	// Process the page
	if (tf($page, $startTime, $endTime)) {
		++$cntSUCCESS;
	} else {
		++$cntFAILURE;
	}
	unset($page);

	// page_Parser statistics	
	$totalTime = $endTime - $startTime;
	$totalElapsedTime += $totalTime;
	echo " - ".(int)$totalTime." ms\n";

}
echo "\n------------- Statistics ---------------------------------\n";

// Success / Failure
echo "\nSUCCESS: ".$cntSUCCESS."\n";
echo "FAILURE: ".$cntFAILURE."\n";

// Time spent
$avgTimePerPage = "N/A";
if($idx > 0) {
	$avgTimePerPage = $totalElapsedTime / $idx;
}
echo "\nTotal elapsed msec: ".$totalElapsedTime."\n";
echo "Avg msec/page: ".$avgTimePerPage."\n";


// Support functions
///////////////////////

// @return current micro time in milli-seconds
function getMicroTime()
{
	$mtime = microtime(); 
	$mtime = explode(" ",$mtime); 
	$mtime = $mtime[1] + $mtime[0]; 
	return $mtime * 1000;	// Convert to milliseconds
}
{'fraction_non_alphanumeric': 0.2525441885377611, 'fraction_numerical': 0.0034815211569362613, 'mean_word_length': 2.736368184092046, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7529789', 'n_tokens_mistral': 2660, 'n_tokens_neox': 2522, 'n_words': 650}