<commit_before>#!/bin/bash
set -e

# Define list of regions to run in
REGION_LIST="eu-west-1 us-east-1 us-east-2 us-west-2"

# Exit if oss/ent parameters not provided
if [[ "$1" == "" ]]; then
    echo "Usage: $(basename $0) [oss/ent]"
    exit 1
else
    RUN_MODE="$1"
fi

ABSPATH=$(readlink -f "$0")
SCRIPT_DIR=$(dirname "${ABSPATH}")
BUILD_DIR=$(readlink -f "${SCRIPT_DIR}/build")

# Remove existing AMI ID file if present
if [ -f "${BUILD_DIR}/amis.txt" ]; then
    rm -f "${BUILD_DIR}/amis.txt"
fi

# Read build timestamp from file
TIMESTAMP_FILE="${BUILD_DIR}/${RUN_MODE}_build_timestamp.txt"
if [ ! -f "${TIMESTAMP_FILE}" ]; then
    echo 'Cannot find "${TIMESTAMP_FILE}"'
    exit 1
fi
BUILD_TIMESTAMP=$(<"${TIMESTAMP_FILE}")

# Write AMI ID for each region to AMI ID file
for REGION in ${REGION_LIST}; do
    aws ec2 describe-images --region ${REGION} --filters Name=tag:BuildTimestamp,Values=${BUILD_TIMESTAMP},Name=tag:BuildType,Values=production > "${BUILD_DIR}/${REGION}.json"<commit_msg>Fix for AWS' filter syntax being terrible<commit_after>#!/bin/bash
set -e

# Define list of regions to run in
REGION_LIST="eu-west-1 us-east-1 us-east-2 us-west-2"

# Exit if oss/ent parameters not provided
if [[ "$1" == "" ]]; then
    echo "Usage: $(basename $0) [oss/ent]"
    exit 1
else
    RUN_MODE="$1"
fi

ABSPATH=$(readlink -f "$0")
SCRIPT_DIR=$(dirname "${ABSPATH}")
BUILD_DIR=$(readlink -f "${SCRIPT_DIR}/build")

# Remove existing AMI ID file if present
if [ -f "${BUILD_DIR}/amis.txt" ]; then
    rm -f "${BUILD_DIR}/amis.txt"
fi

# Read build timestamp from file
TIMESTAMP_FILE="${BUILD_DIR}/${RUN_MODE}_build_timestamp.txt"
if [ ! -f "${TIMESTAMP_FILE}" ]; then
    echo 'Cannot find "${TIMESTAMP_FILE}"'
    exit 1
fi
BUILD_TIMESTAMP=$(<"${TIMESTAMP_FILE}")

# Write AMI ID for each region to AMI ID file
for REGION in ${REGION_LIST}; do
    aws ec2 describe-images --region ${REGION} --filters "Name=tag:BuildTimestamp,Values=${BUILD_TIMESTAMP}" "Name=tag:BuildType,Values=production" > "${BUILD_DIR}/${REGION}.json"
{'fraction_non_alphanumeric': 0.17010816125860373, 'fraction_numerical': 0.010816125860373648, 'mean_word_length': 4.621546961325967, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '518040', 'n_tokens_mistral': 737, 'n_tokens_neox': 724, 'n_words': 202}