<commit_before><?php
/* Start of Functions */
function ip2bin($ip) {
	/* Converts octal IP addresses to non-deliminated binary
	  e.g. xxx.xxx.xxx.xxx to 11111111111111111111111111111111 */
	$binip = decbin(ip2long($ip));
	return $binip;
}
function bin2ip($bin) {
	/* Converts non-deliminated binary to octal IP addresses
	e.g. 11111111111111111111111111111111 to xxx.xxx.xxx.xxx */
	$out = long2ip(bindec(str_pad($bin, 32, "0", STR_PAD_LEFT)));
	return $out;
}
function subnet($binip,$mask) {
	/* Returns a shortened subnet mask */
	$network = substr($binip, 0, $mask);
	return $network;
}
function prepare_input($net,$min) {
	/* Sorts all input network addressses, convers to binary, and cleans duplicates */
	sort($net);
	$y = 0;
	foreach($net as $i) {
		$x = explode('/',$i);
		if(!isset($x[1])) { $x[1] = '32'; }
		if($x[1] > $min) { $x[1] = $min; }
		$out[$y] = subnet(ip2bin($x[0]),$x[1]);
		$y++;
	}
	$out = array_unique($out, SORT_NUMERIC);
	ksort($out);
	return $out;
}
function prepare_output($net,$notation) {
	/* Takes the input binary array, formats as IP address, and appends the CIDR notation */
	ksort($net);
	$y = 0;
	if($notation == 1){
		foreach($net as $i) {
			$len = strlen($i);
			$out[$y] = bin2ip($i) . '/' . $len;
			$y++;
		}
	}
	if($notation == 2){
		foreach($net as $i) {
			$len = strlen($i);
			$out[$y] = bin2ip($i) . ' ' . bin2ip(str_repeat("1",$len));
			$y++;
		}
	}
	if($notation == 3){
		foreach($net as $i) {
			$len = strlen($i);
			$out[$y] = bin2ip($i) . ' ' . bin2ip(str_repeat("0",$len).str_repeat("1",32-$len));
			$y++;
		}
	}
	$out = array_unique($out);
	return $out;
}
function summarize($net) {
	$x = 0;
	do {
		$in = $net;
		foreach($net as $i) {
			$lookup = substr($i,0,strlen($i) -1);
			$pattern = '/\b' . $lookup . '[0,1]\b/';
			if(count(preg_grep($pattern,$net)) > 1) {
				$net = array_diff($net,preg_grep($pattern,$net));
				$net[] = $lookup;
			}
		}
		$out = $net;
		$x++;
	} while (($in !== $out) and ($x < 30));
	return $net;
}
?>
<commit_msg>Update summary.php<commit_after><?php
/* Start of Functions */
function ip2bin($ip) {
	/* Converts octal IP addresses to non-deliminated binary
	  e.g. xxx.xxx.xxx.xxx to 11111111111111111111111111111111 */
	$a = explode('.',$ip);
	$binip = octet($a[0]) . octet($a[1]) . octet($a[2]) . octet($a[3]);
	return $binip;
}
function bin2ip($bin) {
	/* Converts non-deliminated binary to octal IP addresses
	e.g. 11111111111111111111111111111111 to xxx.xxx.xxx.xxx */
	$ip = str_split(str_pad($bin,32, '0', STR_PAD_RIGHT),8);
	$x = 0;
	foreach($ip as $i) {
		$ip[$x] = bindec($i);
		$x++;
	}
	$out = implode('.',$ip);
	return $out;
}
function octet($octet) {
	/* Pads binary octets
	Really just has it's own function to keep the other one looking clean */
	return str_pad(decbin($octet),8,'0', STR_PAD_LEFT);
}
function prepare_input($net,$min) {
	/* Sorts all input network addressses, convers to binary, and cleans duplicates */
	sort($net);
	$y = 0;
	foreach($net as $i) {
		$x = explode('/',$i);
		if(!isset($x[1])) { $x[1] = '32'; }
		if($x[1] > $min) { $x[1] = $min; }
		$out[$y] = subnet(ip2bin($x[0]),$x[1]);
		$y++;
	}
	$out = array_unique($out, SORT_NUMERIC);
	ksort($out);
	return $out;
}
function prepare_output($net,$notation) {
	/* Takes the input binary array, formats as IP address, and appends the CIDR notation */
	ksort($net);
	$y = 0;
	if($notation == 1){
		foreach($net as $i) {
			$len = strlen($i);
			$out[$y] = bin2ip($i) . '/' . $len;
			$y++;
		}
	}
	if($notation == 2){
		foreach($net as $i) {
			$len = strlen($i);
			$out[$y] = bin2ip($i) . ' ' . bin2ip(str_repeat("1",$len));
			$y++;
		}
	}
	if($notation == 3){
		foreach($net as $i) {
			$len = strlen($i);
			$out[$y] = bin2ip($i) . ' ' . bin2ip(str_repeat("0",$len).str_repeat("1",32-$len));
			$y++;
		}
	}
	$out = array_unique($out);
	return $out;
}
function summarize($net) {
	$x = 0;
	do {
		$in = $net;
		foreach($net as $i) {
			$lookup = substr($i,0,strlen($i) -1);
			$pattern = '/\b' . $lookup . '[0,1]\b/';
			if(count(preg_grep($pattern,$net)) > 1) {
				$net = array_diff($net,preg_grep($pattern,$net));
				$net[] = $lookup;
			}
		}
		$out = $net;
		$x++;
	} while (($in !== $out) and ($x < 30));
	return $net;
}
?>

{'fraction_non_alphanumeric': 0.2220447284345048, 'fraction_numerical': 0.04860794157918759, 'mean_word_length': 3.103932584269663, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3352609', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1793, 'n_words': 385}