<commit_before>#!/bin/bash

set -e

if env | grep -q "NYC_TREES_DEPLOY_DEBUG"; then
  set -x
fi

function get_latest_ubuntu_ami() {
  # 1. Get list of daily Ubuntu AMIs
  # 2. Filter AMIs with EBS SSD instance store, amd64 architecture, and in
  #    AWS_DEFAULT_REGION
  # 3. Filter again by HVM AMIs
  # 4. Sort by date in reverse
  # 5. Take the top row
  # 6. Take the 8th column
  curl -s "http://cloud-images.ubuntu.com/query/trusty/server/released.current.txt" \
    | egrep "ebs-ssd\s+amd64\s+${AWS_DEFAULT_REGION}" \
    | grep "hvm" \
    | sort -k4 -r \
    | head -n1 \
    | cut -f8
}

function get_stack_outputs() {
  aws cloudformation describe-stacks \
    --stack-name "${1}" \
    --query "Stacks[*].Outputs[*].[OutputKey, OutputValue]"
}

function wait_for_stack() {
  local n=0

  until [ $n -ge 30 ]  # 30 * 60 = 1800 = 30 minutes
  do
    echo "Waiting for stack..."

    aws cloudformation describe-stacks \
      --stack-name "${1}" \
      | cut -f"7,8" \
      | egrep -q "(CREATE|UPDATE|ROLLBACK)_COMPLETE" && break

    n=$[$n+1]
    sleep 60
  done

  aws cloudformation describe-stacks --stack-name "${1}"
}

wait_for_elb_health() {
  local n=0

  until [ $n -ge 20 ]  # 20 * 30 = 600 = 10 minutes
  do
    echo "Waiting for ELB health..."

    aws elb describe-instance-health \
      --load-balancer-name "${1}" \
      | cut -f5 \
      | grep -q "InService" && break

    n=$[$n+1]
    sleep 30
  done

  aws elb describe-instance-health --load-balancer-name "${1}"
}

toggle_app_server_stack() {
  AWS_STACKS=$(aws cloudformation describe-stacks | grep STACKS | cut -f7)
  AWS_APP_STACK_OUTPUTS=$(get_stack_outputs "NYCTrees${1}AppServers")

  AWS_ELB_APP_ENDPOINT=$(echo $AWS_APP_STACK_OUTPUTS | grep "AppServerLoadBalancerEndpoint" | cut -d' ' -f2)
  AWS_ELB_APP_HOSTED_ZONE_ID=$(echo $AWS_APP_STACK_OUTPUTS | grep "AppServerLoadBalancerHostedZoneNameID" | cut -d' ' -f4)

  # Build parameters argument
  AWS_STACK_PARAMS="ParameterKey=PublicHostedZone,ParameterValue=${AWS_PUBLIC_HOSTED_ZONE}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=AppSSLCertificateId,ParameterValue=${AWS_SSL_CERTIFICATE_ID}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=AppServerAliasTarget,ParameterValue=${AWS_ELB_APP_ENDPOINT}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=AppServerLoadBalancerHostedZoneNameID,ParameterValue=${AWS_ELB_APP_HOSTED_ZONE_ID}"

  if echo "${AWS_STACKS}" | grep -q "NYCTreesAppHostedZone"; then
    aws cloudformation update-stack \
      --stack-name "NYCTreesAppHostedZone" \
      --template-body "file://troposphere/app_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  else
    aws cloudformation create-stack \
      --stack-name "NYCTreesAppHostedZone" \
      --template-body "file://troposphere/app_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  fi

  wait_for_stack "NYCTreesAppHostedZone"
}

toggle_tile_server_stack() {
  AWS_STACKS=$(aws cloudformation describe-stacks | grep STACKS | cut -f7)
  AWS_VPC_STACK_OUTPUTS=$(get_stack_outputs "NYCTreesVPC")
  AWS_TILE_STACK_OUTPUTS=$(get_stack_outputs "NYCTrees${1}TileServers")

  AWS_VPC_ID=$(echo "${AWS_VPC_STACK_OUTPUTS}" | grep "VpcId" | cut -f2)
  AWS_ELB_TILE_ENDPOINT=$(echo $AWS_TILE_STACK_OUTPUTS | grep "TileServerLoadBalancerEndpoint" | cut -d' ' -f2)
  AWS_ELB_TILE_HOSTED_ZONE_ID=$(echo $AWS_TILE_STACK_OUTPUTS | grep "TileServerLoadBalancerHostedZoneNameID" | cut -d' ' -f4)
  AWS_PRIVATE_HOSTED_ZONE_ID=$(aws route53 list-hosted-zones | grep -B1 "${AWS_VPC_ID}" | grep -v "${AWS_VPC_ID}" | cut -f3 | cut -d'/' -f3)

  # Build parameters argument
  AWS_STACK_PARAMS="ParameterKey=PublicHostedZone,ParameterValue=${AWS_PUBLIC_HOSTED_ZONE}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=GlobalNotificationsARN,ParameterValue=${AWS_SNS_TOPIC}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=TileServerAliasTarget,ParameterValue=${AWS_ELB_TILE_ENDPOINT}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=TileServerLoadBalancerHostedZoneNameID,ParameterValue=${AWS_ELB_TILE_HOSTED_ZONE_ID}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=PrivateHostedZoneId,ParameterValue=${AWS_PRIVATE_HOSTED_ZONE_ID}"

  if echo "${AWS_STACKS}" | grep -q "NYCTreesTileHostedZone"; then
    aws cloudformation update-stack \
      --stack-name "NYCTreesTileHostedZone" \
      --template-body "file://troposphere/tiler_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  else
    aws cloudformation create-stack \
      --stack-name "NYCTreesTileHostedZone" \
      --template-body "file://troposphere/tiler_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  fi

  wait_for_stack "NYCTreesTileHostedZone"
}

function get_latest_internal_ami() {
  AWS_IMAGES=$(aws ec2 describe-images --owner self)

  # No images are owned by requesting user; look for images
  # this user can launch.
  if [[ -z $AWS_IMAGES ]]; then
    AWS_IMAGES=$(aws ec2 describe-images --executable-users self)
  fi

  # 1. Get list of AMIs owned by this account
  # 2. Filter by type (only argument to this function)
  # 3. Filter again for the IMAGES row
  # 4. Sort by AMI name (contains a date created timestamp)
  # 5. Take the top row
  # 6. Take the 4th column
  echo "${AWS_IMAGES}" \
    | grep "${1}" \
    | grep IMAGES \
    | sort -k8 -r \
    | head -n1 \
    | cut -f5
}

function create_ami() {
  packer build \
    -only="${1}" \
    -var "aws_ubuntu_ami=$(get_latest_ubuntu_ami)" \
    packer/template.js
}

case "$1" in

  create-vpc-stack)
    aws cloudformation create-stack \
      --stack-name "NYCTreesVPC" \
      --template-body "file://troposphere/vpc_template.json" \
      --parameters "ParameterKey=KeyName,ParameterValue=${AWS_KEY_NAME}"

    wait_for_stack "NYCTreesVPC"
    ;;<commit_msg>Remove all Ansible role related symlinks<commit_after>#!/bin/bash

set -e

if env | grep -q "NYC_TREES_DEPLOY_DEBUG"; then
  set -x
fi

function get_latest_ubuntu_ami() {
  # 1. Get list of daily Ubuntu AMIs
  # 2. Filter AMIs with EBS SSD instance store, amd64 architecture, and in
  #    AWS_DEFAULT_REGION
  # 3. Filter again by HVM AMIs
  # 4. Sort by date in reverse
  # 5. Take the top row
  # 6. Take the 8th column
  curl -s "http://cloud-images.ubuntu.com/query/trusty/server/released.current.txt" \
    | egrep "ebs-ssd\s+amd64\s+${AWS_DEFAULT_REGION}" \
    | grep "hvm" \
    | sort -k4 -r \
    | head -n1 \
    | cut -f8
}

function get_stack_outputs() {
  aws cloudformation describe-stacks \
    --stack-name "${1}" \
    --query "Stacks[*].Outputs[*].[OutputKey, OutputValue]"
}

function wait_for_stack() {
  local n=0

  until [ $n -ge 30 ]  # 30 * 60 = 1800 = 30 minutes
  do
    echo "Waiting for stack..."

    aws cloudformation describe-stacks \
      --stack-name "${1}" \
      | cut -f"7,8" \
      | egrep -q "(CREATE|UPDATE|ROLLBACK)_COMPLETE" && break

    n=$[$n+1]
    sleep 60
  done

  aws cloudformation describe-stacks --stack-name "${1}"
}

wait_for_elb_health() {
  local n=0

  until [ $n -ge 20 ]  # 20 * 30 = 600 = 10 minutes
  do
    echo "Waiting for ELB health..."

    aws elb describe-instance-health \
      --load-balancer-name "${1}" \
      | cut -f5 \
      | grep -q "InService" && break

    n=$[$n+1]
    sleep 30
  done

  aws elb describe-instance-health --load-balancer-name "${1}"
}

toggle_app_server_stack() {
  AWS_STACKS=$(aws cloudformation describe-stacks | grep STACKS | cut -f7)
  AWS_APP_STACK_OUTPUTS=$(get_stack_outputs "NYCTrees${1}AppServers")

  AWS_ELB_APP_ENDPOINT=$(echo $AWS_APP_STACK_OUTPUTS | grep "AppServerLoadBalancerEndpoint" | cut -d' ' -f2)
  AWS_ELB_APP_HOSTED_ZONE_ID=$(echo $AWS_APP_STACK_OUTPUTS | grep "AppServerLoadBalancerHostedZoneNameID" | cut -d' ' -f4)

  # Build parameters argument
  AWS_STACK_PARAMS="ParameterKey=PublicHostedZone,ParameterValue=${AWS_PUBLIC_HOSTED_ZONE}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=AppSSLCertificateId,ParameterValue=${AWS_SSL_CERTIFICATE_ID}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=AppServerAliasTarget,ParameterValue=${AWS_ELB_APP_ENDPOINT}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=AppServerLoadBalancerHostedZoneNameID,ParameterValue=${AWS_ELB_APP_HOSTED_ZONE_ID}"

  if echo "${AWS_STACKS}" | grep -q "NYCTreesAppHostedZone"; then
    aws cloudformation update-stack \
      --stack-name "NYCTreesAppHostedZone" \
      --template-body "file://troposphere/app_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  else
    aws cloudformation create-stack \
      --stack-name "NYCTreesAppHostedZone" \
      --template-body "file://troposphere/app_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  fi

  wait_for_stack "NYCTreesAppHostedZone"
}

toggle_tile_server_stack() {
  AWS_STACKS=$(aws cloudformation describe-stacks | grep STACKS | cut -f7)
  AWS_VPC_STACK_OUTPUTS=$(get_stack_outputs "NYCTreesVPC")
  AWS_TILE_STACK_OUTPUTS=$(get_stack_outputs "NYCTrees${1}TileServers")

  AWS_VPC_ID=$(echo "${AWS_VPC_STACK_OUTPUTS}" | grep "VpcId" | cut -f2)
  AWS_ELB_TILE_ENDPOINT=$(echo $AWS_TILE_STACK_OUTPUTS | grep "TileServerLoadBalancerEndpoint" | cut -d' ' -f2)
  AWS_ELB_TILE_HOSTED_ZONE_ID=$(echo $AWS_TILE_STACK_OUTPUTS | grep "TileServerLoadBalancerHostedZoneNameID" | cut -d' ' -f4)
  AWS_PRIVATE_HOSTED_ZONE_ID=$(aws route53 list-hosted-zones | grep -B1 "${AWS_VPC_ID}" | grep -v "${AWS_VPC_ID}" | cut -f3 | cut -d'/' -f3)

  # Build parameters argument
  AWS_STACK_PARAMS="ParameterKey=PublicHostedZone,ParameterValue=${AWS_PUBLIC_HOSTED_ZONE}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=GlobalNotificationsARN,ParameterValue=${AWS_SNS_TOPIC}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=TileServerAliasTarget,ParameterValue=${AWS_ELB_TILE_ENDPOINT}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=TileServerLoadBalancerHostedZoneNameID,ParameterValue=${AWS_ELB_TILE_HOSTED_ZONE_ID}"
  AWS_STACK_PARAMS="${AWS_STACK_PARAMS} ParameterKey=PrivateHostedZoneId,ParameterValue=${AWS_PRIVATE_HOSTED_ZONE_ID}"

  if echo "${AWS_STACKS}" | grep -q "NYCTreesTileHostedZone"; then
    aws cloudformation update-stack \
      --stack-name "NYCTreesTileHostedZone" \
      --template-body "file://troposphere/tiler_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  else
    aws cloudformation create-stack \
      --stack-name "NYCTreesTileHostedZone" \
      --template-body "file://troposphere/tiler_hosted_zone_template.json" \
      --parameters ${AWS_STACK_PARAMS}
  fi

  wait_for_stack "NYCTreesTileHostedZone"
}

function get_latest_internal_ami() {
  AWS_IMAGES=$(aws ec2 describe-images --owner self)

  # No images are owned by requesting user; look for images
  # this user can launch.
  if [[ -z $AWS_IMAGES ]]; then
    AWS_IMAGES=$(aws ec2 describe-images --executable-users self)
  fi

  # 1. Get list of AMIs owned by this account
  # 2. Filter by type (only argument to this function)
  # 3. Filter again for the IMAGES row
  # 4. Sort by AMI name (contains a date created timestamp)
  # 5. Take the top row
  # 6. Take the 4th column
  echo "${AWS_IMAGES}" \
    | grep "${1}" \
    | grep IMAGES \
    | sort -k8 -r \
    | head -n1 \
    | cut -f5
}

function create_ami() {
  # Remove all symlinks so that Packer does not complain
  find . -type l -delete

  packer build \
    -only="${1}" \
    -var "aws_ubuntu_ami=$(get_latest_ubuntu_ami)" \
    packer/template.js
}

case "$1" in

  create-vpc-stack)
    aws cloudformation create-stack \
      --stack-name "NYCTreesVPC" \
      --template-body "file://troposphere/vpc_template.json" \
      --parameters "ParameterKey=KeyName,ParameterValue=${AWS_KEY_NAME}"

    wait_for_stack "NYCTreesVPC"
    ;;
{'fraction_non_alphanumeric': 0.12270251872021784, 'fraction_numerical': 0.013955071477195371, 'mean_word_length': 3.9403110550651532, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1055126', 'n_tokens_mistral': 4531, 'n_tokens_neox': 4366, 'n_words': 854}