<commit_before><?php 
require_once("ui.inc");
require_once("utils.inc");
require_once("dbapi.inc");


// Return an array of frame times for each page.
$gN = getParam('n');
$crawl = getParam('crawl');
if ( ! $gN || ! $crawl ) {
	exit();
}


// Ask for twice as many as needed so we can filter out adult sites.
$query = "select pageid, url, wptid, wptrun from $gPagesTable where label='$crawl' and rank > 0 and rank <= " . (2*$gN) . " order by rank asc;";
$result = doQuery($query);

$wptServer = wptServer();
$i = 0;
$msMax = 0;
while ($row = mysql_fetch_assoc($result)) {
	$url = $row['url'];
	if ( ! onBlackList($url) ) {
		$pageid = $row['pageid'];
		$wptid = $row['wptid'];
		$wptrun = $row['wptrun'];

		$xmlurl = "{$wptServer}xmlResult.php?test=$wptid";
		$xmlstr = file_get_contents($xmlurl);
		$xml = new SimpleXMLElement($xmlstr);
		$frames = $xml->data->run[($wptrun - 1)]->firstView->videoFrames;
		if ( $frames->frame ) {
			$sJS = "";
			foreach($frames->frame as $frame) {
				$ms = floatval($frame->time) * 1000;
				$msMax = max($msMax, $ms);
				$sJS .= ( $sJS ? ", " : "" ) . "$ms: 1"; // must NOT end with a comma - poop
			}
			echo "hPages[$pageid] = {" . $sJS . "};\n";
		}
		
		$i++;
		if ( $i >= $gN ) {
			break;
		}
	}
}
mysql_free_result($result);
echo "\nvar msMax = $msMax;\n";
?>
<commit_msg>Add a fallback query for crawls that do not have rank.<commit_after><?php 
require_once("ui.inc");
require_once("utils.inc");
require_once("dbapi.inc");


// Return an array of frame times for each page.
$gN = getParam('n');
$gL = getParam('l');
if ( ! $gN || ! $gL ) {
	exit();
}


// Ask for twice as many as needed so we can filter out adult sites.
$query = "select pageid, url, wptid, wptrun from $gPagesTable where label='$gL' and rank > 0 and rank <= " . (2*$gN) . " order by rank asc;";
$result = doQuery($query);
if ( 0 == mysql_num_rows($result) ) {
	mysql_free_result($result);
	// Older crawls do NOT have values for "rank". Use today's rank.
	$query = "select u.rank, pageid, url, wptid, wptrun from $gPagesTable, $gUrlsTable as u where label='$gL' and u.rank > 0 and u.rank <= " . (2*$gN) . " and urlOrig=url order by u.rank asc;";
	$result = doQuery($query);
}

$wptServer = wptServer();
$i = 0;
$msMax = 0;
while ($row = mysql_fetch_assoc($result)) {
	$url = $row['url'];
	if ( ! onBlackList($url) ) {
		$pageid = $row['pageid'];
		$wptid = $row['wptid'];
		$wptrun = $row['wptrun'];

		$xmlurl = "{$wptServer}xmlResult.php?test=$wptid";
		$xmlstr = file_get_contents($xmlurl);
		$xml = new SimpleXMLElement($xmlstr);
		$frames = $xml->data->run[($wptrun - 1)]->firstView->videoFrames;
		if ( $frames->frame ) {
			$sJS = "";
			foreach($frames->frame as $frame) {
				$ms = floatval($frame->time) * 1000;
				$msMax = max($msMax, $ms);
				$sJS .= ( $sJS ? ", " : "" ) . "$ms: 1"; // must NOT end with a comma - poop
			}
			echo "hPages[$pageid] = {" . $sJS . "};\n";
		}
		
		$i++;
		if ( $i >= $gN ) {
			break;
		}
	}
}
mysql_free_result($result);
echo "\nvar msMax = $msMax;\n";
?>

{'fraction_non_alphanumeric': 0.2123015873015873, 'fraction_numerical': 0.007605820105820105, 'mean_word_length': 3.5625942684766216, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2072252', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1199, 'n_words': 299}