<commit_before><?php
function hexview($data,$wrap = 16, $baseaddress = 0, $charset = null) {
	$output = '         ';
	for ($i = 0; $i < $wrap; $i++)
		$output .= sprintf('%02X ', ($baseaddress+$i)%$wrap);
	$output .= sprintf('<hr>%06X: ', $baseaddress);
	$ascii = '';
	for ($i = 0; isset($data[$i]); $i++) {
		$output .= sprintf('%02X ', ord($data[$i]));
		$ascii .= replacechar($data[$i], $charset);
		if ($i%$wrap == $wrap-1) {
			$output .= sprintf("%s\r\n",$ascii);
			if ($i+1 < strlen($data))
				$output .= sprintf('%06X: ', $baseaddress+$i+1);
			$ascii = '';
		}
	}
	if ($ascii != null)
		$output .= str_repeat(' ', ($wrap*3)-mb_strlen($ascii, 'UTF-8')*3).$ascii;
	return $output;
}

function replacechar($char, $charset) {
	if ($charset == null) {
		if (ord($char) < 0x20)
			return '.';
		else if (ord($char) >= 0x7F)
			return '.';<commit_msg>fix some string alignment issues in hexview<commit_after><?php
function hexview($data,$wrap = 16, $baseaddress = 0, $charset = null) {
	$output = '        ';
	for ($i = 0; $i < $wrap; $i++)
		$output .= sprintf('%02X ', ($baseaddress+$i)%$wrap);
	$output .= sprintf('<hr>%06X: ', $baseaddress);
	$ascii = '';
	$stringlen = 0;
	for ($i = 0; isset($data[$i]); $i++) {
		$output .= sprintf('%02X ', ord($data[$i]));
		$stringlen++;
		$ascii .= replacechar($data[$i], $charset);
		if ($i%$wrap == $wrap-1) {
			$output .= sprintf("%s\r\n",$ascii);
			if ($i < strlen($data))
				$output .= sprintf('%06X: ', $baseaddress+$i+1);
			$ascii = '';
			$stringlen = 0;
		}
	}
	if ($ascii != null)
		$output .= str_repeat(' ', ($wrap-$stringlen)*3).$ascii;
	return $output;
}

function replacechar($char, $charset) {
	if ($charset == null) {
		if (ord($char) < 0x20)
			return '.';
		else if (ord($char) >= 0x7F)
			return '.';
{'fraction_non_alphanumeric': 0.264257481648786, 'fraction_numerical': 0.026538678712591756, 'mean_word_length': 3.776280323450135, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6092037', 'n_tokens_mistral': 781, 'n_tokens_neox': 700, 'n_words': 148}