<commit_before>set -e

USERNAME=$1
USER_EMAIL=$2

if [ "$USERNAME" == "" -o "$USER_EMAIL" == "" ]
then
   echo "Usage: $0 <USERNAME> <USER_EMAIL>"
   exit 1
fi

USERNAME=$1
USER_EMAIL=$2

PROJECT_PREFIX="int_test"

echo "python unit tests"
python setup.py -q test

echo "python3 unit tests"
python3 setup.py -q test

export PROJ="python$PROJECT_PREFIX"
echo "test upload $PROJ"
python -m ddsc upload -p $PROJ ddsc/
python -m ddsc add-user -p $PROJ --email $USER_EMAIL

echo "test download $PROJ"
# test filename conversion
python -m ddsc download -p $PROJ
echo "differences:"
diff --brief -r ddsc/ $PROJ/ddsc/
rm -rf $PROJ

export PROJ2="python3$PROJECT_PREFIX"
echo "test upload $PROJ2"
python3 -m ddsc upload -p $PROJ2 ddsc/
python3 -m ddsc add-user -p $PROJ2 --user $USERNAME
python3 -m ddsc remove-user -p $PROJ2 --user $USERNAME

echo "test download $PROJ2"
rm -rf /tmp/$PROJ2
python3 -m ddsc download -p $PROJ /tmp/$PROJ2
echo "differences:"
diff --brief -r ddsc/ /tmp/$PROJ2/ddsc/

echo "Success check data on portal"<commit_msg>integration test must wait for background processing<commit_after>set -e

USERNAME=$1
USER_EMAIL=$2

if [ "$USERNAME" == "" -o "$USER_EMAIL" == "" ]
then
   echo "Usage: $0 <USERNAME> <USER_EMAIL>"
   exit 1
fi

USERNAME=$1
USER_EMAIL=$2

PROJECT_PREFIX="int_test"

echo "python unit tests"
python setup.py -q test

echo "python3 unit tests"
python3 setup.py -q test

export PROJ="python$PROJECT_PREFIX"
echo "test upload $PROJ"
python -m ddsc upload -p $PROJ ddsc/
python -m ddsc add-user -p $PROJ --email $USER_EMAIL

echo "Waiting for DukeDS background processing"
sleep 30
echo "test download $PROJ"
# test filename conversion
python -m ddsc download -p $PROJ
echo "differences:"
diff --brief -r ddsc/ $PROJ/ddsc/
rm -rf $PROJ

export PROJ2="python3$PROJECT_PREFIX"
echo "test upload $PROJ2"
python3 -m ddsc upload -p $PROJ2 ddsc/
python3 -m ddsc add-user -p $PROJ2 --user $USERNAME
python3 -m ddsc remove-user -p $PROJ2 --user $USERNAME

echo "Waiting for DukeDS background processing"
sleep 30
echo "test download $PROJ2"
rm -rf /tmp/$PROJ2
python3 -m ddsc download -p $PROJ /tmp/$PROJ2
echo "differences:"
diff --brief -r ddsc/ /tmp/$PROJ2/ddsc/

echo "Success check data on portal"
{'fraction_non_alphanumeric': 0.1249438202247191, 'fraction_numerical': 0.021573033707865168, 'mean_word_length': 4.024830699774267, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3783924', 'n_tokens_mistral': 872, 'n_tokens_neox': 838, 'n_words': 293}