<commit_before>#!/bin/bash

showstats() {
	tsvout="$1"

	# XXX: if no answer generated, the answer line may be
	# completely missing
	total="$(wc -l <"$tsvout")"
	if [ "$total" -eq 0 ]; then
		echo 'no answers generated'
		return
	fi

	perfect="$(cat "$tsvout" | cut -f4 | grep '^1\.0' | wc -l)"
	any="$(cat "$tsvout" | cut -f4 | grep -v '^0\.0$' | wc -l)"

	perfectp="$(echo "100*$perfect/$total" | bc -l)"
	anyp="$(echo "100*$any/$total" | bc -l)"

	avgscore="$(echo "($(cat "$tsvout" | cut -f4 | tr '\n' '+')0)/$total" | bc -l)"
	avgtime="$(echo "($(cat "$tsvout" | cut -f2 | tr '\n' '+')0)/$total" | bc -l)"

	printf '%d/%d/%d %.1f%%/%.1f%% avgscore %.3f avgtime %.3f\n' \
		"$perfect" "$any" "$total" "$perfectp" "$anyp" "$avgscore" "$avgtime"
}

if [ "$#" -gt 1 ]; then
	for tsvout; do
		printf '%s ' "$tsvout"
		showstats "$tsvout"
	done

else
	# List all commits that have recorded evaluation
	evaldir=$(dirname "$0")/tsv
	commits=$(echo "$evaldir"/*$1*.tsv | sed 's/[^ ]*-\([^.]*\).tsv/\1/g; s/[uvw]//g')
	git log --no-walk --pretty='tformat:%h %ad %s' --date=short $commits |
		while read commit date subject; do
			for file in "$evaldir"/*"$commit".tsv; do
				case $file in *"$1"*) ;; *) continue;; esac
				name="${file##*/}"; name="$(echo "$name" | cut -d- -f2)"
				commit="${file##*/}"; commit="$(echo "$commit" | cut -d- -f4)"; commit="${commit%.*}"
				printf '% 5s % 8s %s %.20s... ' "${name:0:5}" "$commit" "$date" "$subject"
				showstats "$file"
			done
		done
fi | less -F<commit_msg>data/eval/tsvout-stats.sh: Include MRR (mean reciprocal rank), a standard IR measure<commit_after>#!/bin/bash

showstats() {
	tsvout="$1"

	# XXX: if no answer generated, the answer line may be
	# completely missing
	total="$(wc -l <"$tsvout")"
	if [ "$total" -eq 0 ]; then
		echo 'no answers generated'
		return
	fi

	perfect="$(cat "$tsvout" | cut -f4 | grep '^1\.0' | wc -l)"
	any="$(cat "$tsvout" | cut -f4 | grep -v '^0\.0$' | wc -l)"

	perfectp="$(echo "100*$perfect/$total" | bc -l)"
	anyp="$(echo "100*$any/$total" | bc -l)"

	avgscore="$(echo "($(cat "$tsvout" | cut -f4 | tr '\n' '+')0)/$total" | bc -l)"
	mrr="$(echo "($(cat "$tsvout" | cut -f5 | sed 's/-1/10000/; s/.*/1\/(1+&)/' | tr '\n' '+')0)/$total" | bc -l)"
	avgtime="$(echo "($(cat "$tsvout" | cut -f2 | tr '\n' '+')0)/$total" | bc -l)"

	printf '%d/%d/%d %.1f%%/%.1f%% avgscore %.3f mrr %.3f avgtime %.3f\n' \
		"$perfect" "$any" "$total" "$perfectp" "$anyp" "$avgscore" "$mrr" "$avgtime"
}

if [ "$#" -gt 1 ]; then
	for tsvout; do
		printf '%s ' "$tsvout"
		showstats "$tsvout"
	done

else
	# List all commits that have recorded evaluation
	evaldir=$(dirname "$0")/tsv
	commits=$(echo "$evaldir"/*$1*.tsv | sed 's/[^ ]*-\([^.]*\).tsv/\1/g; s/[uvw]//g')
	git log --no-walk --pretty='tformat:%h %ad %s' --date=short $commits |
		while read commit date subject; do
			for file in "$evaldir"/*"$commit".tsv; do
				case $file in *"$1"*) ;; *) continue;; esac
				name="${file##*/}"; name="$(echo "$name" | cut -d- -f2)"
				commit="${file##*/}"; commit="$(echo "$commit" | cut -d- -f4)"; commit="${commit%.*}"
				printf '% 5s % 8s %s %.20s... ' "${name:0:5}" "$commit" "$date" "$subject"
				showstats "$file"
			done
		done
fi | less -F
{'fraction_non_alphanumeric': 0.27108621226268287, 'fraction_numerical': 0.025210084033613446, 'mean_word_length': 3.3315363881401616, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7428266', 'n_tokens_mistral': 1475, 'n_tokens_neox': 1323, 'n_words': 366}