<commit_before>#!/usr/bin/env bash

set -e

export MSYS_NO_PATHCONV=1

script=`cd $(dirname $0) && pwd`/`basename $0`
image=$1

docker_dir="`dirname $script`"
ci_dir="`dirname $docker_dir`"
src_dir="`dirname $ci_dir`"
root_dir="`dirname $src_dir`"

objdir=$root_dir/obj
dist=$objdir/build/dist

source "$ci_dir/shared.sh"

if [ -f "$docker_dir/$image/Dockerfile" ]; then
    if [ "$CI" != "" ]; then
      hash_key=/tmp/.docker-hash-key.txt
      rm -f "${hash_key}"
      echo $image >> $hash_key

      cat "$docker_dir/$image/Dockerfile" >> $hash_key
      # Look for all source files involves in the COPY command
      copied_files=/tmp/.docker-copied-files.txt
      rm -f "$copied_files"
      for i in $(sed -n -e 's/^COPY \(.*\) .*$/\1/p' "$docker_dir/$image/Dockerfile"); do
        # List the file names
        find "$docker_dir/$i" -type f >> $copied_files
      done
      # Sort the file names and cat the content into the hash key
      sort $copied_files | xargs cat >> $hash_key

      docker --version >> $hash_key
      cksum=$(sha512sum $hash_key | \
        awk '{print $1}')

      s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
      url="https://$SCCACHE_BUCKET.s3.amazonaws.com/docker/$cksum"
      upload="aws s3 cp - $s3url"

      echo "Attempting to download $url"
      rm -f /tmp/rustci_docker_cache
      set +e
      retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url"
      loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/')
      set -e
      echo "Downloaded containers:\n$loaded_images"
    fi

    dockerfile="$docker_dir/$image/Dockerfile"
    if [ -x /usr/bin/cygpath ]; then
        context="`cygpath -w $docker_dir`"
        dockerfile="`cygpath -w $dockerfile`"
    else
        context="$docker_dir"
    fi
    retry docker \
      build \
      --rm \
      -t rust-ci \
      -f "$dockerfile" \
      "$context"

    if [ "$upload" != "" ]; then
      digest=$(docker inspect rust-ci --format '{{.Id}}')
      echo "Built container $digest"
      if ! grep -q "$digest" <(echo "$loaded_images"); then
        echo "Uploading finished image to $url"
        set +e
        docker history -q rust-ci | \
          grep -v missing | \
          xargs docker save | \
          gzip | \
          $upload
        set -e
      else
        echo "Looks like docker image is the same as before, not uploading"
      fi
      # Record the container image for reuse, e.g. by rustup.rs builds
      info="$dist/image-$image.txt"
      mkdir -p "$dist"
      echo "$url" >"$info"
      echo "$digest" >>"$info"
    fi
elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
    if isCI; then
        echo Cannot run disabled images on CI!
        exit 1
    fi
    # Transform changes the context of disabled Dockerfiles to match the enabled ones
    tar --transform 's#^./disabled/#./#' -C $docker_dir -c . | docker \<commit_msg>Use CDN for ci-caches on download<commit_after>#!/usr/bin/env bash

set -e

export MSYS_NO_PATHCONV=1

script=`cd $(dirname $0) && pwd`/`basename $0`
image=$1

docker_dir="`dirname $script`"
ci_dir="`dirname $docker_dir`"
src_dir="`dirname $ci_dir`"
root_dir="`dirname $src_dir`"

objdir=$root_dir/obj
dist=$objdir/build/dist

source "$ci_dir/shared.sh"

CACHE_DOMAIN="${CACHE_DOMAIN:-ci-caches.rust-lang.org}"

if [ -f "$docker_dir/$image/Dockerfile" ]; then
    if [ "$CI" != "" ]; then
      hash_key=/tmp/.docker-hash-key.txt
      rm -f "${hash_key}"
      echo $image >> $hash_key

      cat "$docker_dir/$image/Dockerfile" >> $hash_key
      # Look for all source files involves in the COPY command
      copied_files=/tmp/.docker-copied-files.txt
      rm -f "$copied_files"
      for i in $(sed -n -e 's/^COPY \(.*\) .*$/\1/p' "$docker_dir/$image/Dockerfile"); do
        # List the file names
        find "$docker_dir/$i" -type f >> $copied_files
      done
      # Sort the file names and cat the content into the hash key
      sort $copied_files | xargs cat >> $hash_key

      docker --version >> $hash_key
      cksum=$(sha512sum $hash_key | \
        awk '{print $1}')

      url="https://$CACHE_DOMAIN/docker/$cksum"

      echo "Attempting to download $url"
      rm -f /tmp/rustci_docker_cache
      set +e
      retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url"
      loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/')
      set -e
      echo "Downloaded containers:\n$loaded_images"
    fi

    dockerfile="$docker_dir/$image/Dockerfile"
    if [ -x /usr/bin/cygpath ]; then
        context="`cygpath -w $docker_dir`"
        dockerfile="`cygpath -w $dockerfile`"
    else
        context="$docker_dir"
    fi
    retry docker \
      build \
      --rm \
      -t rust-ci \
      -f "$dockerfile" \
      "$context"

    if [ "$CI" != "" ]; then
      s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
      upload="aws s3 cp - $s3url"
      digest=$(docker inspect rust-ci --format '{{.Id}}')
      echo "Built container $digest"
      if ! grep -q "$digest" <(echo "$loaded_images"); then
        echo "Uploading finished image to $url"
        set +e
        docker history -q rust-ci | \
          grep -v missing | \
          xargs docker save | \
          gzip | \
          $upload
        set -e
      else
        echo "Looks like docker image is the same as before, not uploading"
      fi
      # Record the container image for reuse, e.g. by rustup.rs builds
      info="$dist/image-$image.txt"
      mkdir -p "$dist"
      echo "$url" >"$info"
      echo "$digest" >>"$info"
    fi
elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
    if isCI; then
        echo Cannot run disabled images on CI!
        exit 1
    fi
    # Transform changes the context of disabled Dockerfiles to match the enabled ones
    tar --transform 's#^./disabled/#./#' -C $docker_dir -c . | docker \
{'fraction_non_alphanumeric': 0.14474129184984783, 'fraction_numerical': 0.006932702062901589, 'mean_word_length': 2.273381294964029, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7109511', 'n_tokens_mistral': 2117, 'n_tokens_neox': 2001, 'n_words': 564}