<commit_before><?php



function parseText($text)
{
	global $parseStatus, $postNoSmilies, $postNoBr, $postPoster;
	
	//Parse smilies and such
	if($parseStatus <= 1)
		$text = htmlspecialchars($text);
		
	if($parseStatus == 0)
	{
		if(!$postNoBr)
			$text = nl2br($text);
		
		$text = postDoReplaceText($text);
	}
	
	return $text;
}

$tagParseStatus = array(
	"ul" => 1,
	"ol" => 1,
	"li" => 0,

	"table" => 1,
	"td" => 0,
	"th" => 0,

	"img" => 2,
	"code" => 2,
	"source" => 2,
	"pre" => 2,
	"style" => 2,
);

$heavyTags = array(
	"code", "source", "pre"
);

$singleTags = array(
	"user", "forum", "thread",
);
$singleHtmlTags = array(
	"p", "br", "li", "img"
);

$goodHtmlTags = array(
	"a", "b", "br", "center", "code", "del", "div", "em", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "i", "img", "li", "ol", "p", "pre", "s", "span", "strong", "style", "sub", "sup", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul"
);


function tokenValidTag($tagname, $bbcode)
{
	global $badTags, $bbcodeCallbacks, $goodHtmlTags;
	
	if(!$bbcode && in_array(trim(strtolower($tagname)), $badTags))
		return false;
	
	if($bbcode && !array_key_exists($tagname, $bbcodeCallbacks))
		return false;
	if(!$bbcode && !in_array($tagname, $goodHtmlTags))
		return false;
	
	return 
		false === strpos($tagname, ">") &&
		false === strpos($tagname, "<") &&
		false === strpos($tagname, "[") &&
		false === strpos($tagname, "]");
}

function parseToken($token)
{
	if(substr($token, 0, 2) == "[/" && substr($token, strlen($token)-1, 1) == "]")
	{
		$tagname = substr($token, 2, strlen($token)-3);

		if(!tokenValidTag($tagname, true))
			return array("type" => 0, "text" => $token);

		return array(
			"type" => 2,
			"tag" => $tagname,
			"text" => $token
		);		
	}
	if(substr($token, 0, 1) == "[" && substr($token, strlen($token)-1, 1) == "]")
	{
		$tagname = substr($token, 1, strlen($token)-2);

		if(!tokenValidTag($tagname, true))
			return array("type" => 0, "text" => $token);

		$arg = "";
		$ind = strpos($tagname, "=");
		if($ind)
		{
			$arg = substr($tagname, $ind+1);
			$tagname = substr($tagname, 0, $ind);
		}
		
		return array(
			"type" => 1,
			"tag" => $tagname,
			"text" => $token,
			"attributes" => $arg
		);
	}
	if(substr($token, 0, 2) == "</" && substr($token, strlen($token)-1, 1) == ">")
	{
		$tagname = substr($token, 2, strlen($token)-3);
		if(!tokenValidTag($tagname, false))
			return array("type" => 0, "text" => $token);
		return array(
			"type" => 4,
			"tag" => $tagname,
			"text" => $token
		);		
	}
	if(substr($token, 0, 1) == "<" && substr($token, strlen($token)-1, 1) == ">")
	{
		$tagname = substr($token, 1, strlen($token)-2);
		if(!tokenValidTag($tagname, false))
			return array("type" => 0, "text" => $token);
		$arg = "";
		$ind = strpos($tagname, " ");
		if($ind)
		{
			$arg = substr($tagname, $ind+1);
			$tagname = substr($tagname, 0, $ind);
		}
		
		return array(
			"type" => 3,
			"tag" => $tagname,
			"text" => $token,
			"attributes" => $arg
		);
	}
	return array(
		"type" => 0,
		"text" => $token
	);
}

function parse($parenttoken)
{
	global $tokens, $tokenPtr, $heavyTags, $singleTags, $singleHtmlTags, $tagParseStatus, $parseStatus, $bbcodeCallbacks;
	
	$contents = "";
	$finished = false;
	
	$textContents = "";
	<commit_msg>Whoops, how could I screw up this?<commit_after><?php



function parseText($text)
{
	global $parseStatus, $postNoSmilies, $postNoBr, $postPoster;
	
	//Parse smilies and such
	if($parseStatus <= 1)
		$text = htmlspecialchars($text);
		
	if($parseStatus == 0)
	{
		if(!$postNoBr)
			$text = nl2br($text);
		
		$text = postDoReplaceText($text);
	}
	
	return $text;
}

$tagParseStatus = array(
	"ul" => 1,
	"ol" => 1,
	"li" => 0,

	"table" => 1,
	"td" => 0,
	"th" => 0,

	"img" => 2,
	"code" => 2,
	"source" => 2,
	"pre" => 2,
	"style" => 2,
);

$heavyTags = array(
	"code", "source", "pre"
);

$singleTags = array(
	"user", "forum", "thread",
);
$singleHtmlTags = array(
	"p", "br", "li", "img"
);

$goodHtmlTags = array(
	"a", "b", "br", "center", "code", "del", "div", "em", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "i", "img", "li", "ol", "p", "pre", "s", "span", "strong", "style", "sub", "sup", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul"
);


function tokenValidTag($tagname, $bbcode)
{
	global $badTags, $bbcodeCallbacks, $goodHtmlTags;
	
	if(!$bbcode && in_array(trim(strtolower($tagname)), $badTags))
		return false;
	
	if($bbcode && !array_key_exists($tagname, $bbcodeCallbacks))
			return false;

	if(!$bbcode && !in_array($tagname, $goodHtmlTags))
		return false;
	
	return 
		false === strpos($tagname, ">") &&
		false === strpos($tagname, "<") &&
		false === strpos($tagname, "[") &&
		false === strpos($tagname, "]");
}

function parseToken($token)
{
	if(substr($token, 0, 2) == "[/" && substr($token, strlen($token)-1, 1) == "]")
	{
		$tagname = substr($token, 2, strlen($token)-3);

		if(!tokenValidTag($tagname, true))
			return array("type" => 0, "text" => $token);

		return array(
			"type" => 2,
			"tag" => $tagname,
			"text" => $token
		);		
	}
	if(substr($token, 0, 1) == "[" && substr($token, strlen($token)-1, 1) == "]")
	{
		$tagname = substr($token, 1, strlen($token)-2);

		$arg = "";
		$ind = strpos($tagname, "=");
		if($ind)
		{
			$arg = substr($tagname, $ind+1);
			$tagname = substr($tagname, 0, $ind);
		}

		if(!tokenValidTag($tagname, true))
			return array("type" => 0, "text" => $token);
		
		return array(
			"type" => 1,
			"tag" => $tagname,
			"text" => $token,
			"attributes" => $arg
		);
	}
	if(substr($token, 0, 2) == "</" && substr($token, strlen($token)-1, 1) == ">")
	{
		$tagname = substr($token, 2, strlen($token)-3);
		if(!tokenValidTag($tagname, false))
			return array("type" => 0, "text" => $token);
		return array(
			"type" => 4,
			"tag" => $tagname,
			"text" => $token
		);		
	}
	if(substr($token, 0, 1) == "<" && substr($token, strlen($token)-1, 1) == ">")
	{
		$tagname = substr($token, 1, strlen($token)-2);
		$arg = "";
		$ind = strpos($tagname, " ");
		if($ind)
		{
			$arg = substr($tagname, $ind+1);
			$tagname = substr($tagname, 0, $ind);
		}
		
		if(!tokenValidTag($tagname, false))
			return array("type" => 0, "text" => $token);
		return array(
			"type" => 3,
			"tag" => $tagname,
			"text" => $token,
			"attributes" => $arg
		);
	}
	return array(
		"type" => 0,
		"text" => $token
	);
}

function parse($parenttoken)
{
	global $tokens, $tokenPtr, $heavyTags, $singleTags, $singleHtmlTags, $tagParseStatus, $parseStatus, $bbcodeCallbacks;
	
	$contents = "";
	$finished = false;
	
	$textContents = "";
	
{'fraction_non_alphanumeric': 0.25452333890831685, 'fraction_numerical': 0.017333130606659572, 'mean_word_length': 3.9052945563012678, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3985014', 'n_tokens_mistral': 2992, 'n_tokens_neox': 2714, 'n_words': 572}