<commit_before>#!/bin/bash
#
# Run legacy table regression testing
#

if [ -z "$TESTBASE" ] ; then
	echo TESTBASE needs to be passed via the env
	exit 1
fi

. $TESTBASE/test-functions

DETOX=$1
INLINE=$(dirname $DETOX)/inline-detox

locale -a

# ----------------------------------------------------------

INPUT=$(printf "\t \$ &")

# ----------------------------------------------------------

OUTPUT="tab_dollar_and_"
LANG=C.UTF-8
LC_ALL=$LANG
LC_CTYPE=$LANG

CHECK=$(echo "$INPUT" | $INLINE -f detoxrc.manpage)
if [ "$CHECK" != "$OUTPUT" ] ; then
	echo "Man page is a lie.  LANG=$LANG"
	locale
	exit 1
fi

# ----------------------------------------------------------

OUTPUT="tab_dollar_and_"
LANG=de_DE.UTF-8
LC_ALL=$LANG
LC_CTYPE=$LANG

CHECK=$(echo "$INPUT" | $INLINE -f detoxrc.manpage)
if [ "$CHECK" != "$OUTPUT" ] ; then
	echo "Man page is a lie.  LANG=$LANG"
	locale
	exit 1
fi

# ----------------------------------------------------------

OUTPUT="tab_money_and_"
LANG=en_US.UTF-8
LC_ALL=$LANG
LC_CTYPE=$LANG

CHECK=$(echo "$INPUT" | $INLINE -f detoxrc.manpage)
if [ "$CHECK" != "$OUTPUT" ] ; then
	echo "Man page is a lie.  LANG=$LANG"
	locale
	exit 1
fi<commit_msg>Fix language testing on Travis (#22)<commit_after>#!/bin/bash
#
# Run legacy table regression testing
#

if [ -z "$TESTBASE" ] ; then
	echo TESTBASE needs to be passed via the env
	exit 1
fi

. $TESTBASE/test-functions

DETOX=$1
INLINE=$(dirname $DETOX)/inline-detox

# ----------------------------------------------------------

INPUT=$(printf "\t \$ &")

# ----------------------------------------------------------

WHICH=C.UTF-8

FOUND=$(locale -a | grep -c '^C[.]')

if [ "$FOUND" -gt 0 ] ; then
	OUTPUT="tab_dollar_and_"
	LANG=$WHICH
	LC_ALL=$WHICH
	LC_CTYPE=$WHICH

	CHECK=$(echo "$INPUT" | $INLINE -f detoxrc.manpage)
	if [ "$CHECK" != "$OUTPUT" ] ; then
		echo "Man page is a lie. LANG=$WHICH"
		locale
		exit 1
	fi
else
	echo "skipping LANG=$WHICH ... not installed"
fi

# ----------------------------------------------------------

WHICH=de_DE.UTF-8

FOUND=$(locale -a | grep -c '^de_DE')

if [ "$FOUND" -gt 0 ] ; then
	OUTPUT="tab_dollar_and_"
	LANG=$WHICH
	LC_ALL=$WHICH
	LC_CTYPE=$WHICH

	CHECK=$(echo "$INPUT" | $INLINE -f detoxrc.manpage)
	if [ "$CHECK" != "$OUTPUT" ] ; then
		echo "Man page is a lie. LANG=$WHICH"
		locale
		exit 1
	fi
else
	echo "skipping LANG=$WHICH ... not installed"
fi

# ----------------------------------------------------------

WHICH=en_US.UTF-8

FOUND=$(locale -a | grep -c '^en_US')

if [ "$FOUND" -gt 0 ] ; then
	OUTPUT="tab_money_and_"
	LANG=$WHICH
	LC_ALL=$WHICH
	LC_CTYPE=$WHICH

	CHECK=$(echo "$INPUT" | $INLINE -f detoxrc.manpage)
	if [ "$CHECK" != "$OUTPUT" ] ; then
		echo "Man page is a lie. LANG=$WHICH"
		locale
		exit 1
	fi
else
	echo "skipping LANG=$WHICH ... not installed"
fi
{'fraction_non_alphanumeric': 0.3052594171997157, 'fraction_numerical': 0.007462686567164179, 'mean_word_length': 1.9140786749482401, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2986209', 'n_tokens_mistral': 1047, 'n_tokens_neox': 951, 'n_words': 260}