<commit_before>#!/usr/bin/env bash

#Default source file
typeset sourceFile="input-file.txt"
#Default target file
typeset targetFile="output-file.txt"

#Default reference file
typeset -r referenceFilePrefix="ref-"
typeset refTargetFile="${referenceFilePrefix}${targetFile}"

#------------------------------------------------------
#
# Check the generated file by comparing it with an expected file
#
# $1 -> executed command
# $2 -> expected file name
# $3 -> generated file name
#
#------------------------------------------------------
function checkGeneratedFile {

	typeset -r executedCommand="${1}"
	typeset -r expectedFile="${2}"
	typeset -r generatedFile="${3}"

	cmp "${expectedFile}" "${generatedFile}"
	if [[ ${?} -ne 0 ]]
	then
		echo "Error checking file after ${executedCommand}" 1>&2
		exit 1
	fi
}

#------------------------------------------------------
#
# Print the executed command after arguments replacements
#
# $1 -> raw executed command
# $2 -> input file name
# $3... -> command arguments
#
#------------------------------------------------------
function printExecutedCommand {

	typeset -r inputFile="${2}"
	typeset executedCommand=$( echo "${1}" | sed -e 's/"${sourceFile}"/'"${inputFile}"'/g' )

	shift 2

	typeset -i argNumber=1
	while [[ ! -z "${1}" ]]
	do
		executedCommand=$( echo "${executedCommand}" | sed -e 's/'\''"${arg'"${argNumber}"'}"'\''/'"${1}"'/g' -e 's/"${arg'"${argNumber}"'}"/'"${1}"'/g' -e 's/'\''${arg'"${argNumber}"'}'\''/'"${1}"'/g' )
		shift
		(( argNumber ++ ))
	done

	echo -n "${executedCommand}"
}

#------------------------------------------------------
#
# Test all commands in the associated -commands.sh file
#
# $1... -> commands arguments
#
#------------------------------------------------------
function testCommands {

	typeset -r commandsShell=$( dirname "${0}" )/$( basename "${0}" | sed 's/.sh$/-commands.sh/' )

	#Map arguements as argX
	typeset -i argNumber=1
	for arg in ${*}
	do
		eval arg${argNumber}="'${arg}'"
		(( argNumber ++ ))
	done

	#List one-line commands to execute and test
	while read line
	do
		executedCommand=$( printExecutedCommand "${line}" "${sourceFile}" ${*})
		echo "Executing: ${executedCommand}"
		eval "${line}" > "${targetFile}"
		checkGeneratedFile "${executedCommand}" "${refTargetFile}" "${targetFile}"
	done < "${commandsShell}"<commit_msg>Add OK/KO after execution of each command<commit_after>#!/usr/bin/env bash

#Default source file
typeset sourceFile="input-file.txt"
#Default target file
typeset targetFile="output-file.txt"

#Default reference file
typeset -r referenceFilePrefix="ref-"
typeset refTargetFile="${referenceFilePrefix}${targetFile}"

#------------------------------------------------------
#
# Check the generated file by comparing it with an expected file
#
# $1 -> executed command
# $2 -> expected file name
# $3 -> generated file name
#
#------------------------------------------------------
function checkGeneratedFile {

	typeset -r executedCommand="${1}"
	typeset -r expectedFile="${2}"
	typeset -r generatedFile="${3}"

	cmp "${expectedFile}" "${generatedFile}"
	if [[ ${?} -ne 0 ]]
	then
		echo -e " [\033[31mKO\033[0m]" 1>&2
		exit 1
	else
		echo -e " [\033[32mOK\033[0m]" 1>&2
	fi
}

#------------------------------------------------------
#
# Print the executed command after arguments replacements
#
# $1 -> raw executed command
# $2 -> input file name
# $3... -> command arguments
#
#------------------------------------------------------
function printExecutedCommand {

	typeset -r inputFile="${2}"
	typeset executedCommand=$( echo "${1}" | sed -e 's/"${sourceFile}"/'"${inputFile}"'/g' )

	shift 2

	typeset -i argNumber=1
	while [[ ! -z "${1}" ]]
	do
		executedCommand=$( echo "${executedCommand}" | sed -e 's/'\''"${arg'"${argNumber}"'}"'\''/'"${1}"'/g' -e 's/"${arg'"${argNumber}"'}"/'"${1}"'/g' -e 's/'\''${arg'"${argNumber}"'}'\''/'"${1}"'/g' )
		shift
		(( argNumber ++ ))
	done

	echo -n "${executedCommand}"
}

#------------------------------------------------------
#
# Test all commands in the associated -commands.sh file
#
# $1... -> commands arguments
#
#------------------------------------------------------
function testCommands {

	typeset -r commandsShell=$( dirname "${0}" )/$( basename "${0}" | sed 's/.sh$/-commands.sh/' )

	#Map arguements as argX
	typeset -i argNumber=1
	for arg in ${*}
	do
		eval arg${argNumber}="'${arg}'"
		(( argNumber ++ ))
	done

	#List one-line commands to execute and test
	while read line
	do
		executedCommand=$( printExecutedCommand "${line}" "${sourceFile}" ${*})
		echo -n "Executing: ${executedCommand}"
		eval "${line}" > "${targetFile}"
		checkGeneratedFile "${executedCommand}" "${refTargetFile}" "${targetFile}"
	done < "${commandsShell}"
{'fraction_non_alphanumeric': 0.3138455044322499, 'fraction_numerical': 0.014774166314900802, 'mean_word_length': 2.353857041755131, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1975754', 'n_tokens_mistral': 1482, 'n_tokens_neox': 1393, 'n_words': 386}