<commit_before>#!/bin/sh
# $Id: tstcql.sh,v 1.3 2004-10-01 11:43:43 adam Exp $
srcdir=${srcdir:-.}
oIFS="$IFS"
IFS='
'
secno=0
testno=0
comment=0
ecode=0
test -f ${srcdir}/cqlsample || exit 1
test -d cql || mkdir cql
for f in `cat ${srcdir}/cqlsample`; do
	if echo $f | grep '^#' >/dev/null; then
		comment=1
	else
		if test "$comment" = "1"; then
			secno=`expr $secno + 1`
			testno=0
		fi
		comment=0
		testno=`expr $testno + 1`
		OUT1=${srcdir}/cql/$secno.$testno.out
		ERR1=${srcdir}/cql/$secno.$testno.err
		OUT2=cql/$secno.$testno.out.tmp
		ERR2=cql/$secno.$testno.err.tmp
		DIFF=cql/$secno.$testno.diff
		../util/cql2xcql "$f" >$OUT2 2>$ERR2
		if test -f $OUT1 -a -f $ERR1; then
			if diff $OUT1 $OUT2 >$DIFF; then
				rm $DIFF
			else
				echo "diff out $secno $testno $f"
				cat $DIFF
				ecode=1	
			fi
			if diff $ERR1 $ERR2 >$DIFF; then
				rm $DIFF
			else
				echo "diff err $secno $testno $f"
				cat $DIFF
				ecode=1<commit_msg>Clean up temp CQL test files<commit_after>#!/bin/sh
# $Id: tstcql.sh,v 1.4 2004-11-16 22:44:31 adam Exp $
srcdir=${srcdir:-.}
oIFS="$IFS"
IFS='
'
secno=0
testno=0
comment=0
ecode=0
test -f ${srcdir}/cqlsample || exit 1
test -d cql || mkdir cql
for f in `cat ${srcdir}/cqlsample`; do
	if echo $f | grep '^#' >/dev/null; then
		comment=1
	else
		if test "$comment" = "1"; then
			secno=`expr $secno + 1`
			testno=0
		fi
		comment=0
		testno=`expr $testno + 1`
		OUT1=${srcdir}/cql/$secno.$testno.out
		ERR1=${srcdir}/cql/$secno.$testno.err
		OUT2=cql/$secno.$testno.out.tmp
		ERR2=cql/$secno.$testno.err.tmp
		DIFF=cql/$secno.$testno.diff
		../util/cql2xcql "$f" >$OUT2 2>$ERR2
		if test -f $OUT1 -a -f $ERR1; then
			if diff $OUT1 $OUT2 >$DIFF; then
				rm $DIFF
				rm $OUT2
			else
				echo "diff out $secno $testno $f"
				cat $DIFF
				ecode=1	
			fi
			if diff $ERR1 $ERR2 >$DIFF; then
				rm $DIFF
				rm $ERR2
			else
				echo "diff err $secno $testno $f"
				cat $DIFF
				ecode=1
{'fraction_non_alphanumeric': 0.17260981912144702, 'fraction_numerical': 0.04547803617571059, 'mean_word_length': 2.951020408163265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7436731', 'n_tokens_mistral': 1033, 'n_tokens_neox': 900, 'n_words': 222}