<commit_before>[ -d .git/refs/tags ] || mkdir -p .git/refs/tags

:> sed.script

# Answer the sha1 has associated with the tag. The tag must exist in .git or .git/refs/tags
tag()
{
	_tag=$1
	[ -f .git/refs/tags/$_tag ] || error "tag: \"$_tag\" does not exist"
	cat .git/refs/tags/$_tag
}

# Generate a commit using the text specified to make it unique and the tree
# named by the tag specified.
unique_commit()
{
	_text=$1
        _tree=$2
	shift 2
    	echo $_text | git-commit-tree $(tag $_tree) "$@"
}

# Save the output of a command into the tag specified. Prepend
# a substitution script for the tag onto the front of sed.script
save_tag()
{
	_tag=$1	
	[ -n "$_tag" ] || error "usage: save_tag tag commit-args ..."
	shift 1
    	"$@" >.git/refs/tags/$_tag

        echo "s/$(tag $_tag)/$_tag/g" > sed.script.tmp
	cat sed.script >> sed.script.tmp
	rm sed.script
	mv sed.script.tmp sed.script
}

# Replace unhelpful sha1 hashses with their symbolic equivalents 
entag()
{
	sed -f sed.script
}

# Execute a command after first saving, then setting the GIT_AUTHOR_EMAIL
# tag to a specified value. Restore the original value on return.
as_author()
{
	_author=$1
	shift 1
        _save=$GIT_AUTHOR_EMAIL

	export GIT_AUTHOR_EMAIL="$_author"
	"$@"
	if test -z "$_save"
	then
		unset GIT_AUTHOR_EMAIL
	else
		export GIT_AUTHOR_EMAIL="$_save"
	fi
}

commit_date()
{
        _commit=$1
	git-cat-file commit $_commit | sed -n "s/^committer .*> \([0-9]*\) .*/\1/p" 
}

on_committer_date()
{
    _date=$1
    shift 1
    GIT_COMMITTER_DATE=$_date "$@"
}

# Execute a command and suppress any error output.
hide_error()
{
	"$@" 2>/dev/null
}

check_output()
{
	_name=$1
	shift 1
	if eval "$*" | entag > $_name.actual
	then
		diff $_name.expected $_name.actual
	else
		return 1;
	fi
}

# Turn a reasonable test description into a reasonable test name.
# All alphanums translated into -'s which are then compressed and stripped
# from front and back.
name_from_description()
{
        tr "'" '-' | tr '~`!@#$%^&*()_+={}[]|\;:"<>,/? ' '-' | tr -s '-' | tr '[A-Z]' '[a-z]' | sed "s/^-*//;s/-*\$//"
}

<commit_msg>t6000lib: workaround a possible dash bug<commit_after>[ -d .git/refs/tags ] || mkdir -p .git/refs/tags

:> sed.script

# Answer the sha1 has associated with the tag. The tag must exist in .git or .git/refs/tags
tag()
{
	_tag=$1
	[ -f .git/refs/tags/$_tag ] || error "tag: \"$_tag\" does not exist"
	cat .git/refs/tags/$_tag
}

# Generate a commit using the text specified to make it unique and the tree
# named by the tag specified.
unique_commit()
{
	_text=$1
        _tree=$2
	shift 2
    	echo $_text | git-commit-tree $(tag $_tree) "$@"
}

# Save the output of a command into the tag specified. Prepend
# a substitution script for the tag onto the front of sed.script
save_tag()
{
	_tag=$1	
	[ -n "$_tag" ] || error "usage: save_tag tag commit-args ..."
	shift 1
    	"$@" >.git/refs/tags/$_tag

        echo "s/$(tag $_tag)/$_tag/g" > sed.script.tmp
	cat sed.script >> sed.script.tmp
	rm sed.script
	mv sed.script.tmp sed.script
}

# Replace unhelpful sha1 hashses with their symbolic equivalents 
entag()
{
	sed -f sed.script
}

# Execute a command after first saving, then setting the GIT_AUTHOR_EMAIL
# tag to a specified value. Restore the original value on return.
as_author()
{
	_author=$1
	shift 1
        _save=$GIT_AUTHOR_EMAIL

	export GIT_AUTHOR_EMAIL="$_author"
	"$@"
	if test -z "$_save"
	then
		unset GIT_AUTHOR_EMAIL
	else
		export GIT_AUTHOR_EMAIL="$_save"
	fi
}

commit_date()
{
        _commit=$1
	git-cat-file commit $_commit | sed -n "s/^committer .*> \([0-9]*\) .*/\1/p" 
}

on_committer_date()
{
    _date=$1
    shift 1
    export GIT_COMMITTER_DATE="$_date"
    "$@"
    unset GIT_COMMITTER_DATE
}

# Execute a command and suppress any error output.
hide_error()
{
	"$@" 2>/dev/null
}

check_output()
{
	_name=$1
	shift 1
	if eval "$*" | entag > $_name.actual
	then
		diff $_name.expected $_name.actual
	else
		return 1;
	fi
}

# Turn a reasonable test description into a reasonable test name.
# All alphanums translated into -'s which are then compressed and stripped
# from front and back.
name_from_description()
{
        tr "'" '-' | tr '~`!@#$%^&*()_+={}[]|\;:"<>,/? ' '-' | tr -s '-' | tr '[A-Z]' '[a-z]' | sed "s/^-*//;s/-*\$//"
}


{'fraction_non_alphanumeric': 0.15436870461466384, 'fraction_numerical': 0.010306863434059498, 'mean_word_length': 3.109720885466795, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169477', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1610, 'n_words': 482}