<commit_before>#!/bin/bash

if [ -x /bin/uname ];then
  UNAME=/bin/uname
else
  UNAME=/usr/bin/uname
fi

SYSTEM=`${UNAME} -s`


VALGRIND_NUMCALLERS=30
if [ "${SYSTEM}" == "Darwin" ];then
  GREP=/usr/bin/grep
else
  GREP=/bin/grep
fi  

VALGRIND=/usr/bin/valgrind
TPUT=/usr/bin/tput

if [ -z "$TMP_DIR" ];then
  TMP_DIR=/tmp/
fi

if [ -z "$USE_VALGRIND" ];then
  USE_VALGRIND=no
fi

if [ -z "$DO_LEAKCHECK" ];then
  DO_LEAKCHECK=no
fi

FILTER_SBML_INPUT_WARNINGS=${FILTER_SBML_IMPORT_WARNINGS:="yes"}
FILTER_RDF_MESSAGES=${FILTER_RDF_MESSAGES:="yes"}


if [ -z "$COPASISE" ];then
  if [ "$SYSTEM" == "Darwin" ];then
    COPASISE=../../../CopasiSE/CopasiSE.app/Contents/MacOS/CopasiSE
  else
    COPASISE=../../../CopasiSE/CopasiSE
  fi
fi

if [ -z "$COPASISE_OPTIONS" ];then
  COPASISE_OPTIONS="--nologo --verbose"
fi

VALGRIND_OPTIONS="--num-callers=${VALGRIND_NUMCALLERS} ${VALGRIND_OPTIONS}"

source ./utilities.sh

function test_roundtrip_single_file
{
    # we pass all the commands as parameters so we don't have to check
    # for their existence in each call. Thoses checks have to be done by the
    # caller
    if [ $# != 2 ];then
        echo "Error. test_import_single_file expects exactly two arguments.";
        return 1;
    fi
    SBML_FILE=$1
    OUTPUT_DIR=$2
    NAME=${SBML_FILE##*/};
    NAME=${NAME%%.xml};
    SBML_OUT_FILE=${NAME}-reexport.xml
    OUTPUT_FILE=${NAME}.roundtrip.out
    ERROR_FILE=${NAME}.roundtrip.err
    VALGRIND_LOG=${NAME}.roundtrip.log
    COMMAND="${COPASISE} ${COPASISE_OPTIONS} --importSBML ${SBML_FILE} --exportSBML ${OUTPUT_DIR}/${SBML_OUT_FILE}"
    if [ "$USE_VALGRIND" == "yes" ];then
       COMMAND="${VALGRIND} ${VALGRIND_OPTIONS} --log-file=${OUTPUT_DIR}/${VALGRIND_LOG} ${COMMAND}"
    fi
    $COMMAND > ${OUTPUT_DIR}/${OUTPUT_FILE} 2> ${OUTPUT_DIR}/${ERROR_FILE} || return 1;
    # check if the CPS file has been generated and has a size different from 0
    if [ ! -s ${OUTPUT_DIR}/${SBML_OUT_FILE} ];then
        return 1;
    fi
    # check if filtering of warnings has been enabled and if yes,
    # do the filtering
    if [ "${FILTER_SBML_IMPORT_WARNINGS}" == "yes" ];then
      mv ${OUTPUT_DIR}/${ERROR_FILE} ${OUTPUT_DIR}/${ERROR_FILE}.unfiltered;
      ${SED} -e '/>.*(filtered)/,/^ *\. *$/d' ${OUTPUT_DIR}/${ERROR_FILE}.unfiltered > ${OUTPUT_DIR}/${ERROR_FILE}
    fi
    # check if the error file has a size different from 0
    if [ -s ${OUTPUT_DIR}/${ERROR_FILE} ];then
        if [ "$USE_VALGRIND" == "yes" ];then
          # additional valgrind checks
          check_valgrind_errors ${OUTPUT_DIR}/${VALGRIND_LOG}
          return $?
        else
            return 2;
        fi
    fi
    # todo check if valgrind found an error
    if [ "$USE_VALGRIND" == "yes" ];then
      check_valgrind_errors ${OUTPUT_DIR}/${VALGRIND_LOG}
      return $?
    fi
    return 0
}

function test_roundtrip_files
{
    OUTPUT_DIR=$TMP_DIR
    FILENAME=$1
    while [ -n "$FILENAME" ];do
      # check if the file exists
      if [ ! -r $FILENAME ];then
          echo "$FILENAME does not exist, is not a regular file or is not readable."
      else
        echo -n "Roundtripping $FILENAME ... ";
        NAME=${FILENAME##*/};
        NAME=${NAME%%.xml};
        test_roundtrip_single_file ${FILENAME} ${OUTPUT_DIR}
        case $? in
        0 )
            echo -n -e '\E[32;47mOK';
            ${TPUT} sgr0;
            echo -n -e "\n";
            ;;
        1 )
            echo -n -e '\E[31;47mFAILED';
            ${TPUT} sgr0;
            echo -n -e "\n";
            ;;
        2 )
            echo -n -e '\E[33;47mSUCCEDED';
            ${TPUT} sgr0;
            echo -e "\nThere was additional output from COPASI.\nCheck ${OUTPUT_DIR}/${NAME}.roundtrip.err for details.";
            ;;
        102 ) 
            ${TPUT} sgr0;
            echo -e "\nValgrind reported errors. Check ${OUTPUT_DIR}/${NAME}.roundtrip.log for details.";
            ;;
        103 ) 
            echo -n -e '\E[33;47mSUCCEDED';
            ${TPUT} sgr0;
            echo -e "\nValgrind reported errors and memory leaks. Check ${OUTPUT_DIR}/${NAME}.roundtrip.log.";
            ;;
        104 ) 
            echo -e -n '\E[33;47mSUCCEDED';
            ${TPUT} sgr0;
            echo -e "\nValgrind reported memory leaks. Check ${OUTPUT_DIR}/${NAME}.roundtrip.log for details.";
            ;;
        * )
            echo -n -e '\E[31;47mFAILED';
            ${TPUT} sgr0;
            echo -e "\nAn unknown error code was reported from test_roundtrip_single_file.";
            ;;
        esac
      fi
      shift
      FILENAME=$1
    done
}

# enable leak checking in valgrind if requested
if [ "${DO_LEAKCHECK}" == "yes" ];then
    VALGRIND_OPTIONS="${VALGRIND_OPTIONS} --leak-check=full"
fi

# check if valgrind is there if needed
if [ ${USE_VALGRIND} == "yes" ]; then
   # todo check if the valgrind executable exists
   check_executable $VALGRIND || exit 1; 
fi

# check if tput is there
check_executable $TPUT || exit 1;

# check if grep is there
check_executable $GREP || exit 1;

# check if CopasiSE is there
check_executable $COPASISE || exit 1;

# check if TMP_DIR is present, a directry and writable
if [ -d $TMP_DIR ];then
    touch ${TMP_DIR}/test.test
    if [ $? != 0 ];then
        echo "Error. Can not write to ${TMP_DIR}."
        exit 1
    else
      rm ${TMP_DIR}/test.test
    fi
else
    echo "Error. ${TMP_DIR} is not a directory."
    exit 1;
fi

test_roundtrip_files $*

<commit_msg>Added some missing lines to the roundtripping test.<commit_after>#!/bin/bash

if [ -x /bin/uname ];then
  UNAME=/bin/uname
else
  UNAME=/usr/bin/uname
fi

SYSTEM=`${UNAME} -s`


VALGRIND_NUMCALLERS=30
if [ "${SYSTEM}" == "Darwin" ];then
  GREP=/usr/bin/grep
  SED=/usr/bin/sed
else
  SED=/bin/sed
  GREP=/bin/grep
fi  

VALGRIND=/usr/bin/valgrind
TPUT=/usr/bin/tput

if [ -z "$TMP_DIR" ];then
  TMP_DIR=/tmp/
fi

if [ -z "$USE_VALGRIND" ];then
  USE_VALGRIND=no
fi

if [ -z "$DO_LEAKCHECK" ];then
  DO_LEAKCHECK=no
fi

FILTER_SBML_INPUT_WARNINGS=${FILTER_SBML_IMPORT_WARNINGS:="yes"}
FILTER_RDF_MESSAGES=${FILTER_RDF_MESSAGES:="yes"}


if [ -z "$COPASISE" ];then
  if [ "$SYSTEM" == "Darwin" ];then
    COPASISE=../../../CopasiSE/CopasiSE.app/Contents/MacOS/CopasiSE
  else
    COPASISE=../../../CopasiSE/CopasiSE
  fi
fi

if [ -z "$COPASISE_OPTIONS" ];then
  COPASISE_OPTIONS="--nologo --verbose"
fi

VALGRIND_OPTIONS="--num-callers=${VALGRIND_NUMCALLERS} ${VALGRIND_OPTIONS}"

source ./utilities.sh

function test_roundtrip_single_file
{
    # we pass all the commands as parameters so we don't have to check
    # for their existence in each call. Thoses checks have to be done by the
    # caller
    if [ $# != 2 ];then
        echo "Error. test_import_single_file expects exactly two arguments.";
        return 1;
    fi
    SBML_FILE=$1
    OUTPUT_DIR=$2
    NAME=${SBML_FILE##*/};
    NAME=${NAME%%.xml};
    SBML_OUT_FILE=${NAME}-reexport.xml
    OUTPUT_FILE=${NAME}.roundtrip.out
    ERROR_FILE=${NAME}.roundtrip.err
    VALGRIND_LOG=${NAME}.roundtrip.log
    COMMAND="${COPASISE} ${COPASISE_OPTIONS} --importSBML ${SBML_FILE} --exportSBML ${OUTPUT_DIR}/${SBML_OUT_FILE}"
    if [ "$USE_VALGRIND" == "yes" ];then
       COMMAND="${VALGRIND} ${VALGRIND_OPTIONS} --log-file=${OUTPUT_DIR}/${VALGRIND_LOG} ${COMMAND}"
    fi
    $COMMAND > ${OUTPUT_DIR}/${OUTPUT_FILE} 2> ${OUTPUT_DIR}/${ERROR_FILE} || return 1;
    # check if the CPS file has been generated and has a size different from 0
    if [ ! -s ${OUTPUT_DIR}/${SBML_OUT_FILE} ];then
        return 1;
    fi
    # check if filtering of warnings has been enabled and if yes,
    # do the filtering
    if [ "${FILTER_SBML_IMPORT_WARNINGS}" == "yes" ];then
      mv ${OUTPUT_DIR}/${ERROR_FILE} ${OUTPUT_DIR}/${ERROR_FILE}.unfiltered;
      ${SED} -e '/>.*(filtered)/,/^ *\. *$/d' ${OUTPUT_DIR}/${ERROR_FILE}.unfiltered > ${OUTPUT_DIR}/${ERROR_FILE}
    fi
    # check if the error file has a size different from 0
    if [ -s ${OUTPUT_DIR}/${ERROR_FILE} ];then
        if [ "$USE_VALGRIND" == "yes" ];then
          # additional valgrind checks
          check_valgrind_errors ${OUTPUT_DIR}/${VALGRIND_LOG}
          return $?
        else
            return 2;
        fi
    fi
    # todo check if valgrind found an error
    if [ "$USE_VALGRIND" == "yes" ];then
      check_valgrind_errors ${OUTPUT_DIR}/${VALGRIND_LOG}
      return $?
    fi
    return 0
}

function test_roundtrip_files
{
    OUTPUT_DIR=$TMP_DIR
    FILENAME=$1
    while [ -n "$FILENAME" ];do
      # check if the file exists
      if [ ! -r $FILENAME ];then
          echo "$FILENAME does not exist, is not a regular file or is not readable."
      else
        echo -n "Roundtripping $FILENAME ... ";
        NAME=${FILENAME##*/};
        NAME=${NAME%%.xml};
        test_roundtrip_single_file ${FILENAME} ${OUTPUT_DIR}
        case $? in
        0 )
            echo -n -e '\E[32;47mOK';
            ${TPUT} sgr0;
            echo -n -e "\n";
            ;;
        1 )
            echo -n -e '\E[31;47mFAILED';
            ${TPUT} sgr0;
            echo -n -e "\n";
            ;;
        2 )
            echo -n -e '\E[33;47mSUCCEDED';
            ${TPUT} sgr0;
            echo -e "\nThere was additional output from COPASI.\nCheck ${OUTPUT_DIR}/${NAME}.roundtrip.err for details.";
            ;;
        102 ) 
            ${TPUT} sgr0;
            echo -e "\nValgrind reported errors. Check ${OUTPUT_DIR}/${NAME}.roundtrip.log for details.";
            ;;
        103 ) 
            echo -n -e '\E[33;47mSUCCEDED';
            ${TPUT} sgr0;
            echo -e "\nValgrind reported errors and memory leaks. Check ${OUTPUT_DIR}/${NAME}.roundtrip.log.";
            ;;
        104 ) 
            echo -e -n '\E[33;47mSUCCEDED';
            ${TPUT} sgr0;
            echo -e "\nValgrind reported memory leaks. Check ${OUTPUT_DIR}/${NAME}.roundtrip.log for details.";
            ;;
        * )
            echo -n -e '\E[31;47mFAILED';
            ${TPUT} sgr0;
            echo -e "\nAn unknown error code was reported from test_roundtrip_single_file.";
            ;;
        esac
      fi
      shift
      FILENAME=$1
    done
}

# enable leak checking in valgrind if requested
if [ "${DO_LEAKCHECK}" == "yes" ];then
    VALGRIND_OPTIONS="${VALGRIND_OPTIONS} --leak-check=full"
fi

# check if valgrind is there if needed
if [ ${USE_VALGRIND} == "yes" ]; then
   # todo check if the valgrind executable exists
   check_executable $VALGRIND || exit 1; 
fi

# check if tput is there
check_executable $TPUT || exit 1;

# check if grep is there
check_executable $GREP || exit 1;

# check if sed is there
check_executable $SED || exit 1;

# check if CopasiSE is there
check_executable $COPASISE || exit 1;

# check if TMP_DIR is present, a directry and writable
if [ -d $TMP_DIR ];then
    touch ${TMP_DIR}/test.test
    if [ $? != 0 ];then
        echo "Error. Can not write to ${TMP_DIR}."
        exit 1
    else
      rm ${TMP_DIR}/test.test
    fi
else
    echo "Error. ${TMP_DIR} is not a directory."
    exit 1;
fi

test_roundtrip_files $*


{'fraction_non_alphanumeric': 0.14284421928713587, 'fraction_numerical': 0.011850913696399493, 'mean_word_length': 2.3931860036832413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4736908', 'n_tokens_mistral': 4064, 'n_tokens_neox': 3842, 'n_words': 1002}