<commit_before>#!/bin/bash
set -e

echo "--- :buildkite: Downloading webpack artifacts"

rm -rf "dist"
mkdir -p "dist"
buildkite-agent artifact download "dist/*" "dist/"

echo "--- :s3: Deploying frontend to $DIST_S3_URL"

s3cmd put -P --recursive --verbose --force --no-preserve "dist/" "$DIST_S3_URL"

echo "--- :wastebasket: Cleaning up.."

rm -rf "tmp/verify"
mkdir -p "tmp/verify"

echo "--- :earth_asia: Downloading files from $FRONTEND_HOST"

# Download the files in assets.json
for url in $(cat dist/assets.json | jq -r '.[].js'); do
  # Make sure the URL is prefixed with https (if it's just got //)
  URL=$(echo $url | sed -e 's/^\/\//https:\/\//')

  pushd "tmp/verify" >> /dev/null
  echo "Downloading $URL"
  curl -OsS "$URL"
  popd >> /dev/null
done

echo "--- :mag: Verifiying the files uploaded match the downloads"

# Check that the files are the same
for f in tmp/verify/*; do
  NAME=$(basename $f)

  DOWNLOADED=$(openssl md5 "tmp/verify/$NAME" | sed 's/^.* //')
  LOCAL=$(openssl md5 "dist/$NAME" | sed 's/^.* //')

  if [[ "$DOWNLOADED" == "$LOCAL" ]]; then
    echo "✅  $NAME"
  else
    echo "❌  $NAME"
    echo "tmp/verify/$NAME isn't the same as dist/$NAME"
    echo "Downloaded version: $DOWNLOADED"
    echo "Local version: $LOCAL"
    exit 1
  fi
done

echo "--- :s3: Deploying manifests to $MANIFEST_S3_URL"

s3cmd put -P --recursive --verbose --force --no-preserve "dist/assets.json" "$MANIFEST_S3_URL/$BUILDKITE_COMMIT/"
<commit_msg>Updated "deploy manifest" header<commit_after>#!/bin/bash
set -e

echo "--- :buildkite: Downloading webpack artifacts"

rm -rf "dist"
mkdir -p "dist"
buildkite-agent artifact download "dist/*" "dist/"

echo "--- :s3: Deploying frontend to $DIST_S3_URL"

s3cmd put -P --recursive --verbose --force --no-preserve "dist/" "$DIST_S3_URL"

echo "--- :wastebasket: Cleaning up.."

rm -rf "tmp/verify"
mkdir -p "tmp/verify"

echo "--- :earth_asia: Downloading files from $FRONTEND_HOST"

# Download the files in assets.json
for url in $(cat dist/assets.json | jq -r '.[].js'); do
  # Make sure the URL is prefixed with https (if it's just got //)
  URL=$(echo $url | sed -e 's/^\/\//https:\/\//')

  pushd "tmp/verify" >> /dev/null
  echo "Downloading $URL"
  curl -OsS "$URL"
  popd >> /dev/null
done

echo "--- :mag: Verifiying the files uploaded match the downloads"

# Check that the files are the same
for f in tmp/verify/*; do
  NAME=$(basename $f)

  DOWNLOADED=$(openssl md5 "tmp/verify/$NAME" | sed 's/^.* //')
  LOCAL=$(openssl md5 "dist/$NAME" | sed 's/^.* //')

  if [[ "$DOWNLOADED" == "$LOCAL" ]]; then
    echo "✅  $NAME"
  else
    echo "❌  $NAME"
    echo "tmp/verify/$NAME isn't the same as dist/$NAME"
    echo "Downloaded version: $DOWNLOADED"
    echo "Local version: $LOCAL"
    exit 1
  fi
done

echo "--- :s3: Deploying manifests to $MANIFEST_S3_URL/$BUILDKITE_COMMIT"

s3cmd put -P --recursive --verbose --force --no-preserve "dist/assets.json" "$MANIFEST_S3_URL/$BUILDKITE_COMMIT/"

{'fraction_non_alphanumeric': 0.17470489038785836, 'fraction_numerical': 0.007419898819561552, 'mean_word_length': 3.1366806136680614, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4851037', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1092, 'n_words': 345}