<commit_before><!-- define ant target "roundtrip" for uni test -->
<project>
	<target name="_roundtrip">

		<!-- setup roundtrip input and output dir -->
		<delete dir="${out.dir}" />
		<mkdir dir="${out.dir}" />

		<!-- run 2 way test -->
		<uploadpo user="bob" apiKey="${bob.key}" dst="${docsUrl}" srcDir="${in.dir}" debug="true" />
		<downloadpo user="bob" apiKey="${bob.key}" src="${docsUrl}" dstDir="${out.dir}" debug="true" />

		
		<!-- diff if input and output are same - disabled for now
		<exec executable="diff" failonerror="true">
			<arg value="- -unified" />
			<arg value="- -recursive" />
			<arg value="${in.dir}" />
			<arg value="${out.dir}" />
		</exec>
		-->
		

	</target>
</project><commit_msg>Changed long diff options to short to work better with XML comments<commit_after><!-- define ant target "roundtrip" for uni test -->
<project>
	<target name="_roundtrip">

		<!-- setup roundtrip input and output dir -->
		<delete dir="${out.dir}" />
		<mkdir dir="${out.dir}" />

		<!-- run 2 way test -->
		<uploadpo user="bob" apiKey="${bob.key}" dst="${docsUrl}" srcDir="${in.dir}" debug="true" />
		<downloadpo user="bob" apiKey="${bob.key}" src="${docsUrl}" dstDir="${out.dir}" debug="true" />

		
		<!-- diff (unified, recursive) to find out if input and output are same 
		- disabled for now
		<exec executable="diff" failonerror="true">
			<arg value="-u" /> 
			<arg value="-r" />
			<arg value="${in.dir}" />
			<arg value="${out.dir}" />
		</exec>
		-->
		

	</target>
</project>
{'fraction_non_alphanumeric': 0.22893165228931653, 'fraction_numerical': 0.0013271400132714001, 'mean_word_length': 3.4222873900293256, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4633007', 'n_tokens_mistral': 559, 'n_tokens_neox': 480, 'n_words': 139}