<commit_before>#!/usr/bin/env bash
set -u
#set -x
base=$(cd $(dirname $0)/..; pwd)

CYAN="\033[0;36m"
RED="\033[0;31m"
GREEN="\033[0;32m"
NONE="\033[0m"

function compare {
  test_name="$1"
  val1=$2
  val2=$3
  
  if [[ $val1 != $val2 ]]; then
      echo -e >&2 "${RED}Test FAILED: $test_name: $val1 != $val2${NONE}"
      return 1
  else
      echo -e >&2 "${GREEN}Test succeeded: $test_name: $val1 == $val2${NONE}"
      return 0
  fi
}

errors=0
tests=0

for iRun in {0..3}; do
    for iRef in {0..3}; do
        refs=$base/reg-test/set1/set1.ref${iRef}
        hyps=$base/reg-test/set1/set1.out.${iRun}
        me=$($base/multeval.sh eval --metrics bleu --refs $refs --hyps-baseline $hyps --bleu.verbosity 1 --verbosity 1 --boot-samples 1 |& awk '/BLEU: OptRun 0/{printf "%.2f\n",$6}')
        mb=$($base/reg-test/multi-bleu.perl $refs < $hyps |& awk -F'( |,)' '{print $3}')
        compare "Set 1 -- multi-bleu.pl Single Ref $iRef Run $iRun" $me $mb        
        errors=$(( $errors + $? ))
        tests=$(( $tests + 1 ))

        # Create dummy SGML files for NIST mt-eval
        sgml=sgml
        $base/reg-test/write-sgm.py en $hyps $sgml $refs
        nist=$($base/reg-test/mteval-v13m.pl -r $sgml/ref -s $sgml/src -t $sgml/hyps --no-norm -b |& awk '/^BLEU score/{printf "%.2f\n",$4*100}')
        compare "Set 1 -- mteval-v13m.pl Single Ref $iRef Run $iRun" $me $nist
        errors=$(( $errors + $? ))
        tests=$(( $tests + 1 ))
    done
done

for iRun in {0..3}; do
    refs_stem=$base/reg-test/set1/set1.ref
    hyps=$base/reg-test/set1/set1.out.${iRun}
    me=$($base/multeval.sh eval --metrics bleu --refs ${refs_stem}? --hyps-baseline $hyps --bleu.verbosity 1 --verbosity 1 --boot-samples 1 |& awk '/BLEU: OptRun 0/{printf "%.2f\n",$6}')
    mb=$($base/reg-test/multi-bleu.perl $refs_stem < $hyps |& awk -F'( |,)' '{print $3}')
    compare "Set 1 -- multi-bleu.pl MultiRef Run $iRun" $me $mb
    errors=$(( $errors + $? ))
    tests=$(( $tests + 1 ))

    # Create dummy SGML files for NIST mt-eval
    sgml=sgml
    $base/reg-test/write-sgm.py en $hyps $sgml ${refs_stem}?
    nist=$($base/reg-test/mteval-v13m.pl -r $sgml/ref -s $sgml/src -t $sgml/hyps --no-norm -b |& awk '/^BLEU score/{printf "%.2f\n",$4*100}')
    compare "Set 1 -- mteval-v13m.pl MultiRef Run $iRun" $me $nist
    errors=$(( $errors + $? ))
    tests=$(( $tests + 1 ))
done

if (( $errors == 0 )); then
    echo -e >&2 "${GREEN}SUMMARY: Ran $tests tests total${NONE}"
    echo -e >&2 "${GREEN}SUMMARY: All tests passed${NONE}"
else
    echo -e >&2 "${RED}SUMMARY: Ran $tests tests total${NONE}"<commit_msg>More detailed output and error messages in regression tests<commit_after>#!/usr/bin/env bash
set -u
set -x
base=$(cd $(dirname $0)/..; pwd)

CYAN="\033[0;36m"
RED="\033[0;31m"
GREEN="\033[0;32m"
NONE="\033[0m"

function compare {
  test_name="$1"
  val1="$2"
  val2="$3"
  
  if [ -e "$val1" ]; then
      echo -e >&2 "${RED}ERROR: $test_name: val1 empty"
      return 1
  fi

  if [ -e "$val2" ]; then
      echo -e >&2 "${RED}ERROR: $test_name: val2 empty"
      return 1
  fi
  
  if [[ $val1 != $val2 ]]; then
      echo -e >&2 "${RED}Test FAILED: $test_name: $val1 != $val2${NONE}"
      return 1
  else
      echo -e >&2 "${GREEN}Test succeeded: $test_name: $val1 == $val2${NONE}"
      return 0
  fi
}

errors=0
tests=0

for iRun in {0..3}; do
    for iRef in {0..3}; do
        refs=$base/reg-test/set1/set1.ref${iRef}
        hyps=$base/reg-test/set1/set1.out.${iRun}
        me=$($base/multeval.sh eval --metrics bleu --refs $refs --hyps-baseline $hyps --bleu.verbosity 1 --verbosity 1 --boot-samples 1 |& tee /dev/stderr | awk '/BLEU: OptRun 0/{printf "%.2f\n",$6}')
        mb=$($base/reg-test/multi-bleu.perl $refs < $hyps |& tee /dev/stderr | awk -F'( |,)' '{print $3}')
        compare "Set 1 -- multi-bleu.pl Single Ref $iRef Run $iRun" "$me" "$mb"
        errors=$(( $errors + $? ))
        tests=$(( $tests + 1 ))

        # Create dummy SGML files for NIST mt-eval
        sgml=sgml
        $base/reg-test/write-sgm.py en $hyps $sgml $refs
        nist=$($base/reg-test/mteval-v13m.pl -r $sgml/ref -s $sgml/src -t $sgml/hyps --no-norm -b |& tee /dev/stderr | awk '/^BLEU score/{printf "%.2f\n",$4*100}')
        compare "Set 1 -- mteval-v13m.pl Single Ref $iRef Run $iRun" "$me" "$nist"
        errors=$(( $errors + $? ))
        tests=$(( $tests + 1 ))
    done
done

for iRun in {0..3}; do
    refs_stem=$base/reg-test/set1/set1.ref
    hyps=$base/reg-test/set1/set1.out.${iRun}
    me=$($base/multeval.sh eval --metrics bleu --refs ${refs_stem}? --hyps-baseline $hyps --bleu.verbosity 1 --verbosity 1 --boot-samples 1 |& tee /dev/stderr | awk '/BLEU: OptRun 0/{printf "%.2f\n",$6}')
    mb=$($base/reg-test/multi-bleu.perl $refs_stem < $hyps |& tee /dev/stderr | awk -F'( |,)' '{print $3}')
    compare "Set 1 -- multi-bleu.pl MultiRef Run $iRun" $me $mb
    errors=$(( $errors + $? ))
    tests=$(( $tests + 1 ))

    # Create dummy SGML files for NIST mt-eval
    sgml=sgml
    $base/reg-test/write-sgm.py en $hyps $sgml ${refs_stem}?
    nist=$($base/reg-test/mteval-v13m.pl -r $sgml/ref -s $sgml/src -t $sgml/hyps --no-norm -b |& tee /dev/stderr | awk '/^BLEU score/{printf "%.2f\n",$4*100}')
    compare "Set 1 -- mteval-v13m.pl MultiRef Run $iRun" $me $nist
    errors=$(( $errors + $? ))
    tests=$(( $tests + 1 ))
done

if (( $errors == 0 )); then
    echo -e >&2 "${GREEN}SUMMARY: Ran $tests tests total${NONE}"
    echo -e >&2 "${GREEN}SUMMARY: All tests passed${NONE}"
else
    echo -e >&2 "${RED}SUMMARY: Ran $tests tests total${NONE}"
{'fraction_non_alphanumeric': 0.21147982062780268, 'fraction_numerical': 0.03659192825112108, 'mean_word_length': 2.6420640104506856, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6303478', 'n_tokens_mistral': 2429, 'n_tokens_neox': 2258, 'n_words': 569}