<commit_before>replace_newline_return () {
	awk '{if (NR>1) {printf("%s⏎", line);} line=$0;} END { printf("%s", line); }'
}

regex_escape () {
	sed 's/\\/\\\\/g' | sed 's/\[/\\\[/g' | sed 's/\]/\\\]/g' | sed 's/\./\\\./g' | sed 's/\*/\\\*/g' | sed 's/\?/\\\?/g' \
		| sed 's/(/\\\(/g' | sed 's/)/\\\)/g' |  sed 's/"/\\"/g'
}

if [ -z "@USE_CMAKE_KDB_COMMAND@" ]; then
	KDBCOMMAND="@KDB_COMMAND@"
	export PATH="$PATH:`dirname \"$KDBCOMMAND\"`"
else
	KDBCOMMAND="kdb"
fi<commit_msg>Shell Recorder: Prefer `kdb` command in build dir<commit_after>replace_newline_return () {
	awk '{if (NR>1) {printf("%s⏎", line);} line=$0;} END { printf("%s", line); }'
}

regex_escape () {
	sed 's/\\/\\\\/g' | sed 's/\[/\\\[/g' | sed 's/\]/\\\]/g' | sed 's/\./\\\./g' | sed 's/\*/\\\*/g' | sed 's/\?/\\\?/g' \
		| sed 's/(/\\\(/g' | sed 's/)/\\\)/g' |  sed 's/"/\\"/g'
}

if [ -z "@USE_CMAKE_KDB_COMMAND@" ]; then
	KDBCOMMAND="@KDB_COMMAND@"
	export PATH="`dirname \"$KDBCOMMAND\"`:$PATH"
else
	KDBCOMMAND="kdb"
fi
{'fraction_non_alphanumeric': 0.34839357429718876, 'fraction_numerical': 0.004016064257028112, 'mean_word_length': 2.5863309352517985, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4609484', 'n_tokens_mistral': 486, 'n_tokens_neox': 482, 'n_words': 86}