<commit_before>#!/usr/bin/env bash

TOP_DIR=$(readlink -f `dirname "${BASH_SOURCE[0]}"` | grep -o '.*/oshinko-s2i/')
BUILD_DIR=$TOP_DIR/pyspark-build

# See what spark version the image build used
fullname=$(find $BUILD_DIR -name spark-[0-9.]*\.tgz)

# Download the same version to use as a binary build input
filename=$(basename $fullname)
version=$(echo $filename | cut -d '-' -f2)
mkdir -p $TOP_DIR/test/e2e/resources/spark-inputs
pushd $TOP_DIR/test/e2e/resources/spark-inputs
if ! [ -f "spark-$version-bin-hadoop2.7.tgz" ]; then
    wget https://archive.apache.org/dist/spark/spark-$version/spark-$version-bin-hadoop2.7.tgz
fi
if ! [ -f "spark-$version-bin-hadoop2.7.tgz.md5" ]; then
    wget https://archive.apache.org/dist/spark/spark-$version/spark-$version-bin-hadoop2.7.tgz.md5
fi
echo "spark-$version-bin-hadoop2.7.tgz: FF FF FF FF FF FF CA FE  BE EF CA FE BE EF CA FE" > spark-$version-bin-hadoop2.7.tgz.bad
popd

# Make a fake tarball that is missing spark-submit
mkdir -p $TOP_DIR/test/e2e/resources/spark-inputs-no-submit
pushd $TOP_DIR/test/e2e/resources/spark-inputs-no-submit
mkdir spark-$version-bin-hadoop2.7<commit_msg>Fix incomplete test to handle missing md5<commit_after>#!/usr/bin/env bash

TOP_DIR=$(readlink -f `dirname "${BASH_SOURCE[0]}"` | grep -o '.*/oshinko-s2i/')
BUILD_DIR=$TOP_DIR/pyspark-build

# See what spark version the image build used
fullname=$(find $BUILD_DIR -name spark-[0-9.]*\.tgz)

# Download the same version to use as a binary build input
filename=$(basename $fullname)
version=$(echo $filename | cut -d '-' -f2)
mkdir -p $TOP_DIR/test/e2e/resources/spark-inputs
pushd $TOP_DIR/test/e2e/resources/spark-inputs
if ! [ -f "spark-$version-bin-hadoop2.7.tgz" ]; then
    wget https://archive.apache.org/dist/spark/spark-$version/spark-$version-bin-hadoop2.7.tgz
fi
if ! [ -f "spark-$version-bin-hadoop2.7.tgz.md5" ]; then
    wget https://archive.apache.org/dist/spark/spark-$version/spark-$version-bin-hadoop2.7.tgz.md5
    if [ "$?" -ne 0 ]; then
	echo "Failed to download md5, calculating from tarball for the test"
	md5sum spark-$version-bin-hadoop2.7.tgz > spark-$version-bin-hadoop2.7.tgz.md5
    fi
fi
echo "spark-$version-bin-hadoop2.7.tgz: FF FF FF FF FF FF CA FE  BE EF CA FE BE EF CA FE" > spark-$version-bin-hadoop2.7.tgz.bad
popd

# Make a fake tarball that is missing spark-submit
mkdir -p $TOP_DIR/test/e2e/resources/spark-inputs-no-submit
pushd $TOP_DIR/test/e2e/resources/spark-inputs-no-submit
mkdir spark-$version-bin-hadoop2.7
{'fraction_non_alphanumeric': 0.156839149618933, 'fraction_numerical': 0.023666265543521862, 'mean_word_length': 4.008032128514056, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6592941', 'n_tokens_mistral': 990, 'n_tokens_neox': 914, 'n_words': 212}