<commit_before><?php 
include "config.php";
$content=file_get_contents($file);
$a=token_get_all($content); ?>
<html>
<head>
	<script type="text/javascript" src="token.js"></script>
	<script type="text/javascript" src="test.min.js"></script>
	<?php/*<script type="application/PHPJS" id="test" style="display:none"><?php echo $content?></script>*/?>
	<script type="text/javascript">
		function DOMReady(f){
		  if (/(?!.*?compatible|.*?webkit)^mozilla|opera/i.test(navigator.userAgent)){ // Feeling dirty yet?
			document.addEventListener("DOMContentLoaded", f, false);
		  }else{
			window.setTimeout(f,0);
		  }
		} 
		function replaceAll(st,rp,rpm)
		{
			while(st.indexOf(rp)>-1) st=st.replace(rp,rpm);
			return st;
		}
		function fixToken(token)
		{
			token=replaceAll(token,"\n","\\n");
			token=replaceAll(token,"\t","\\t");
			token=replaceAll(token,"\r","\\r");
			token=replaceAll(token," ","&nbsp;");
			return token;
		}
		DOMReady(function(){			
			var cont=file_get_contents("s.php");<commit_msg>Fix file reference for tokenizer testing<commit_after><?php 
include "config.php";
$content=file_get_contents($file);
$a=token_get_all($content); ?>
<html>
<head>
	<script type="text/javascript" src="token_get_all.js"></script>
	<script type="text/javascript" src="test.min.js"></script>
	<?php/*<script type="application/PHPJS" id="test" style="display:none"><?php echo $content?></script>*/?>
	<script type="text/javascript">
		function DOMReady(f){
		  if (/(?!.*?compatible|.*?webkit)^mozilla|opera/i.test(navigator.userAgent)){ // Feeling dirty yet?
			document.addEventListener("DOMContentLoaded", f, false);
		  }else{
			window.setTimeout(f,0);
		  }
		} 
		function replaceAll(st,rp,rpm)
		{
			while(st.indexOf(rp)>-1) st=st.replace(rp,rpm);
			return st;
		}
		function fixToken(token)
		{
			token=replaceAll(token,"\n","\\n");
			token=replaceAll(token,"\t","\\t");
			token=replaceAll(token,"\r","\\r");
			token=replaceAll(token," ","&nbsp;");
			return token;
		}
		DOMReady(function(){			
			var cont=file_get_contents("s.php");
{'fraction_non_alphanumeric': 0.21508515815085158, 'fraction_numerical': 0.0019464720194647203, 'mean_word_length': 5.192771084337349, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1529938', 'n_tokens_mistral': 811, 'n_tokens_neox': 730, 'n_words': 106}