<commit_before>#!/bin/bash
if [ "$1" == "-h" ]; then
cat << USAGE
Description:
Math rank test script to test one test-case.

Examples:
$0 cases/inequality.txt
USAGE
exit
fi

[ $# -ne 1 ] && echo 'bad arg.' && exit

# make tmp directory
tmp_corpus=./tmp-corpus
mkdir -p $tmp_corpus

# set variable
testcase="$1"

# clear expect-rank file
> expect-rank.log

# extract sections
head -1 "$testcase" > $tmp_corpus/query
tail -n +3 "$testcase" > $tmp_corpus/docs

# print test query
echo "[test query: $(cat "$tmp_corpus/query")]"

# for each doc in docs
i=0
while read line
do
	# parse line
	prefix=$(echo "$line" | awk '{print $1}')
	tex=$(echo "$line" | cut -d' ' -f 2- )

	if [ "$prefix" != 'HIT' -a  "$prefix" != 'NOT' ]; then
		echo "invalid test-case format."
		exit 2
	fi;

	# generate corpus files
	echo "[imath]$tex[/imath]" > $tmp_corpus/doc${i}
	./run/txt2json.ln $tmp_corpus/doc${i} > /dev/null

	# generate expect ranking
	if [ "$prefix" == 'HIT' ]; then
		echo "expect   doc${i}: $tex"
		echo "doc${i}" >> expect-rank.log
	else
		echo "unexpect doc${i}: $tex"
	fi;

	# increment i
	let 'i=i+1'
done << EOF
$(cat "$tmp_corpus/docs")
EOF

# index those corpus files
./run/indexer.ln -e -p $tmp_corpus > /dev/null

# search test query
qry=$(cat "$tmp_corpus/query")
echo "[searching $qry]"

# echo full search command
set -x
./run/searcher.ln -e -n -i ./tmp -m "$qry" > srch-res.log
set +x

# extract search results
grep 'URL:' srch-res.log | awk '{print $2}' | \
	awk -F/ '{print $NF}' > rank.log

echo '[rank]'
cat rank.log

echo '[diff]'
diff expect-rank.log rank.log > /dev/null

if [ $? -eq 0 ]; then
	# exact what we expect
	echo 'same'
	exit 0
else
	# different from what we expect
	echo 'different'
	exit 1
fi;
<commit_msg>Fix two problems in math-rank-test.sh<commit_after>#!/bin/bash
if [ "$1" == "-h" ]; then
cat << USAGE
Description:
Math rank test script to test one test-case.

Examples:
$0 cases/inequality.txt
USAGE
exit
fi

[ $# -ne 1 ] && echo 'bad arg.' && exit

# clear potential old corpus
tmp_corpus=./tmp-corpus
rm -rf $tmp_corpus
mkdir -p $tmp_corpus

# set variable
testcase="$1"

# clear expect-rank file
> expect-rank.log

# extract sections
head -1 "$testcase" > $tmp_corpus/query
tail -n +3 "$testcase" > $tmp_corpus/docs

# print test query
echo "[test query: $(cat "$tmp_corpus/query")]"

# for each doc in docs
i=0
while read -r line
do
	# parse line
	prefix=$(echo "$line" | awk '{print $1}')
	tex=$(echo "$line" | cut -d' ' -f 2- )

	if [ "$prefix" != 'HIT' -a  "$prefix" != 'NOT' ]; then
		echo "invalid test-case format."
		exit 2
	fi;

	# generate corpus files
	echo "[imath]$tex[/imath]" > $tmp_corpus/doc${i}
	./run/txt2json.ln $tmp_corpus/doc${i} > /dev/null

	# generate expect ranking
	if [ "$prefix" == 'HIT' ]; then
		echo "expect   doc${i}: $tex"
		echo "doc${i}" >> expect-rank.log
	else
		echo "unexpect doc${i}: $tex"
	fi;

	# increment i
	let 'i=i+1'
done << EOF
$(cat "$tmp_corpus/docs")
EOF

# index those corpus files
./run/indexer.ln -e -p $tmp_corpus > /dev/null

# search test query
qry=$(cat "$tmp_corpus/query")
echo "[searching $qry]"

# echo full search command
set -x
./run/searcher.ln -e -n -i ./tmp -m "$qry" > srch-res.log
set +x

# extract search results
grep 'URL:' srch-res.log | awk '{print $2}' | \
	awk -F/ '{print $NF}' > rank.log

echo '[rank]'
cat rank.log

echo '[diff]'
diff expect-rank.log rank.log > /dev/null

if [ $? -eq 0 ]; then
	# exact what we expect
	echo 'same'
	exit 0
else
	# different from what we expect
	echo 'different'
	exit 1
fi;

{'fraction_non_alphanumeric': 0.17630385487528344, 'fraction_numerical': 0.009070294784580499, 'mean_word_length': 3.3729863692688973, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1516466', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1380, 'n_words': 441}