<commit_before>#!/bin/bash

# This creates a pachyderm cluster in AWS for testing. This is a thin wrapper around
# etc/deploy/aws.sh, but it uses the same state store bucket for all tests, so that
# kops clusters created for testing can always be enumerated and deleted.

## Parse command-line flags

STATE_STORE=s3://pachyderm-travis-state-store-v1
REGION=-
ZONE=us-west-1b
OP=-

# Process args
new_opt="$( getopt --long="create,delete:,delete-all,list,zone:" -- ${0} "${@}" )"
[[ "$?" -eq 0 ]] || exit 1
eval "set -- ${new_opt}"

while true; do
  case "${1}" in
    --delete-all)
      OP=delete-all
      shift
      ;;
    --list)
      kops --state=${STATE_STORE} get clusters
      exit 0  # Shortcut
      ;;
    --delete)
      OP=delete
      NAME="${2}"
      shift 2
      ;;
    --create)
      OP=create
      shift
      ;;
    --zone)
      ZONE="${2}"
      shift 2
      ;;
    --)
      shift
      break
      ;;
  esac
done

len_zone_minus_one="$(( ${#ZONE} - 1 ))"
REGION=${ZONE:0:${len_zone_minus_one}}

echo -e "Region: ${REGION}\nZone: ${ZONE}"

# No need to authenticate with kops, as auth creds are already in environment variables
# in travis
set -x
case "${OP}" in
  create)
    aws_sh="$(realpath "$(dirname "${0}")/../../deploy/aws.sh")"
    sudo "${aws_sh}" --region=${REGION} --zone=${ZONE} --state=${STATE_STORE} --no-metrics
    ;;
  delete)
    kops --state=${STATE_STORE} delete cluster --name=${NAME} --yes<commit_msg>Apply --use-cloudfront flag to testing deploy script<commit_after>#!/bin/bash

# This creates a pachyderm cluster in AWS for testing. This is a thin wrapper around
# etc/deploy/aws.sh, but it uses the same state store bucket for all tests, so that
# kops clusters created for testing can always be enumerated and deleted.

## Parse command-line flags

STATE_STORE=s3://pachyderm-travis-state-store-v1
REGION=-
ZONE=us-west-1b
OP=-
CLOUDFRONT=

# Process args
new_opt="$( getopt --long="create,delete:,delete-all,list,zone:,use-cloudfront" -- ${0} "${@}" )"
[[ "$?" -eq 0 ]] || exit 1
eval "set -- ${new_opt}"

while true; do
  case "${1}" in
    --delete-all)
      OP=delete-all
      shift
      ;;
    --list)
      kops --state=${STATE_STORE} get clusters
      exit 0  # Shortcut
      ;;
    --delete)
      OP=delete
      NAME="${2}"
      shift 2
      ;;
    --create)
      OP=create
      shift
      ;;
    --zone)
      ZONE="${2}"
      shift 2
      ;;
    --use-cloudfront)
      # Default is not to provide the flag
      CLOUDFRONT="--use-cloudfront"
      shift
      ;;
    --)
      shift
      break
      ;;
  esac
done

len_zone_minus_one="$(( ${#ZONE} - 1 ))"
REGION=${ZONE:0:${len_zone_minus_one}}

echo -e "Region: ${REGION}\nZone: ${ZONE}"

# No need to authenticate with kops, as auth creds are already in environment variables
# in travis
set -x
case "${OP}" in
  create)
    aws_sh="$(realpath "$(dirname "${0}")/../../deploy/aws.sh")"
    sudo "${aws_sh}" --region=${REGION} --zone=${ZONE} --state=${STATE_STORE} --no-metrics $CLOUDFRONT
    ;;
  delete)
    kops --state=${STATE_STORE} delete cluster --name=${NAME} --yes
{'fraction_non_alphanumeric': 0.17041800643086816, 'fraction_numerical': 0.00964630225080386, 'mean_word_length': 2.5635738831615122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '781542', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1069, 'n_words': 299}