<commit_before>#!/bin/bash

if [ "$#" -ne "2" ]; then
    echo "Usage: $0 location destination"
    exit 1
fi

/usr/bin/sudo /usr/bin/rkt run \
    --net=host --dns=host \
    --trust-keys-from-https ${awscli_image} \
    --volume=tmp,kind=host,source=/tmp --mount=volume=tmp,target=/tmp \
    --set-env="LOCATION=$1" \
    --exec=/bin/bash -- -c '
        REGION=$(wget -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/[a-zA-Z]$//')
        /usr/bin/aws --region=$${REGION} s3 cp s3://$${LOCATION} /tmp/$${LOCATION//\//+}
    '

/usr/bin/sudo mv /tmp/$${1//\//+} $2<commit_msg>modules/aws/ignition/resources/s3-puller.sh: add retries to avoid races<commit_after>#!/bin/bash

if [ "$#" -ne "2" ]; then
    echo "Usage: $0 location destination"
    exit 1
fi

/usr/bin/sudo /usr/bin/rkt run \
    --net=host --dns=host \
    --trust-keys-from-https ${awscli_image} \
    --volume=tmp,kind=host,source=/tmp --mount=volume=tmp,target=/tmp \
    --set-env="LOCATION=$1" \
    --exec=/bin/bash -- -c '
        REGION=$(wget -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/[a-zA-Z]$//')
        until /usr/bin/aws --region=$${REGION} s3 cp s3://$${LOCATION} /tmp/$${LOCATION//\//+}; do
          echo "Could not pull from S3, retrying in 5 seconds ($i)"
          sleep 5
        done
    '

/usr/bin/sudo mv /tmp/$${1//\//+} $2
{'fraction_non_alphanumeric': 0.23082437275985662, 'fraction_numerical': 0.031541218637992835, 'mean_word_length': 2.2616822429906542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7139654', 'n_tokens_mistral': 589, 'n_tokens_neox': 549, 'n_words': 102}