<commit_before><?xml version="1.0"?>
<application>
	<name>deltaforce</name>
	<category>General file manipulation</category>
	<description>merges two sorted flat files with some different columns</description>
	<options>
		<option has_argument="0" required="0">
			<name>help</name>
			<short>h</short>
			<long>help</long>
			<description>print this message and exit</description>
		</option>
		<option has_argument="0" required="0">
			<name>version</name>
			<short>V</short>
			<long>version</long>
			<description>print version info and exit</description>
		</option>
		<option has_argument="1" required="0">
			<name>verbose</name>
			<short>v</short>
			<long>verbose</long>
			<description>print verbose messages during execution</description>
		</option>
		<option has_argument="1" required="0">
			<name>delim</name>
			<short>d</short>
			<long>delim</long>
			<description>delimiting string for both input files</description>
		</option>
		<option has_argument="1" required="0">
			<name>outfile</name>
			<short>o</short>
			<long>outfile</long>
			<description>name of file for output</description>
		</option>
	</options>
	<details>
		<item></item>
	</details>
	<example><![CDATA[
# add new impressions to a state file containing each user's
# most recent impressions.  both files should contain a header line,
# and the key must be the first field.

$ cat state.log

User-ID	Time
57646080047293658810-29-2007-11:39:17
57646081134276441911-02-2007-19:55:28
57646081326886364010-26-2007-15:39:56
57646081411116490410-30-2007-16:53:36
57646082078170678411-09-2007-12:29:18
57646082435411647111-09-2007-12:40:31

$ gzdog Impression_673394_11-10-2007.log.gz |
	reorder -f 2,1 |
	(read h; echo $h; sort -k 1,1 -k 2r,2r) \
	> delta.log
$ cat delta.log

User-ID	Time
57646081134276441911-10-2007-19:55:28
57646082078170678411-10-2007-12:29:18
57646082435413456711-10-2007-12:40:31

$ <b>deltaforce</b> state.log delta.log

User-ID	Time
576460800472936588	10-29-2007-11:39:17
576460811342764419	11-10-2007-19:55:28
576460813268863640	10-26-2007-15:39:56
576460814111164904	10-30-2007-16:53:36
576460820781706784	11-10-2007-12:29:18
576460824354116471	11-09-2007-12:40:31
576460824354134567	11-10-2007-12:40:31

]]></example>
</application><commit_msg>apps/utils/deltaforce/deltaforce.crush.xml - change delimiter in the example to be more utf8/xml-friendly<commit_after><?xml version="1.0"?>
<application>
	<name>deltaforce</name>
	<category>General file manipulation</category>
	<description>merges two sorted flat files with some different columns</description>
	<options>
		<option has_argument="0" required="0">
			<name>help</name>
			<short>h</short>
			<long>help</long>
			<description>print this message and exit</description>
		</option>
		<option has_argument="0" required="0">
			<name>version</name>
			<short>V</short>
			<long>version</long>
			<description>print version info and exit</description>
		</option>
		<option has_argument="1" required="0">
			<name>verbose</name>
			<short>v</short>
			<long>verbose</long>
			<description>print verbose messages during execution</description>
		</option>
		<option has_argument="1" required="0">
			<name>delim</name>
			<short>d</short>
			<long>delim</long>
			<description>delimiting string for both input files</description>
		</option>
		<option has_argument="1" required="0">
			<name>outfile</name>
			<short>o</short>
			<long>outfile</long>
			<description>name of file for output</description>
		</option>
	</options>
	<details>
		<item></item>
	</details>
	<example><![CDATA[
# add new impressions to a state file containing each user's
# most recent impressions.  both files should contain a header line,
# and the key must be the first field.

$ cat state.log

User-ID	Time
576460800472936588	10-29-2007-11:39:17
576460811342764419	11-02-2007-19:55:28
576460813268863640	10-26-2007-15:39:56
576460814111164904	10-30-2007-16:53:36
576460820781706784	11-09-2007-12:29:18
576460824354116471	11-09-2007-12:40:31

$ gzdog Impression_673394_11-10-2007.log.gz |
	reorder -f 2,1 |
	(read h; echo $h; sort -k 1,1 -k 2r,2r) \
	> delta.log
$ cat delta.log

User-ID	Time
576460811342764419	11-10-2007-19:55:28
576460820781706784	11-10-2007-12:29:18
576460824354134567	11-10-2007-12:40:31

$ <b>deltaforce</b> -d "\t" state.log delta.log

User-ID	Time
576460800472936588	10-29-2007-11:39:17
576460811342764419	11-10-2007-19:55:28
576460813268863640	10-26-2007-15:39:56
576460814111164904	10-30-2007-16:53:36
576460820781706784	11-10-2007-12:29:18
576460824354116471	11-09-2007-12:40:31
576460824354134567	11-10-2007-12:40:31

]]></example>
</application>
{'fraction_non_alphanumeric': 0.14795587280986372, 'fraction_numerical': 0.23556132381570408, 'mean_word_length': 4.8979591836734695, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 2, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4657813', 'n_tokens_mistral': 2529, 'n_tokens_neox': 1803, 'n_words': 347}