<commit_before>#!/usr/bin/env bash

#
# build, test and deploy
#

set -o errexit
set -o pipefail
set -o nounset
# set -o xtrace

source helper.sh

# Usage info
show_help() {
    cat << EOF
Usage: ${0##*/} [-hv] [-ci] [-s:]
    Build and deploy everything. You should specify what to deploy (code, infrastructure)
    Specify at least one of -c or -i.

    -c 		build and deploy lambda function. Can be used with -i too.
    -d          delete needed enviroment variables. cleanup. defaults are used.
    -h          display this help and exit
    -i          deploy infrastructure. Can be used with -c too.
    -p          disable personal build/deploy for development 
                normally cloudformation stacks will be suffixed with your local username (3) 
    -v          verbose mode.

    example usage:
    ./deploy.sh -ci
    ./deploy.sh -ci -p      to deploy to production (no personal build)
EOF
}

# A POSIX variable
OPTIND=1         # Reset in case getopts has been used previously in the shell.

# Initialize our own variables:
verbose=false
cleanup=false
DEPLOY_INFRASTRUCTURE=false
DEPLOY_CODE=false
PERSONAL_BUILD=true

while getopts "h?vicdp" opt; do
    case "$opt" in
    h|\?)
        show_help
        exit 0
        ;;
    v)  verbose=true
        ;;
    d)  cleanup=true
        ;;
    i)  DEPLOY_INFRASTRUCTURE=true
        ;;
    c)  DEPLOY_CODE=true
        ;;
    p)  PERSONAL_BUILD=false
        ;;
    esac
done

shift $((OPTIND-1))

if [ ${DEPLOY_INFRASTRUCTURE} == false ] && [ ${DEPLOY_CODE} == false ] && [ ${cleanup} == false ]; then
    show_help
    exit 0
fi

# check preconditions
command -v cf >/dev/null 2>&1 || { echo >&2 "I require cfn-sphere but it's not installed. Install with: pip install cfn-sphere. Aborting."; exit 1; }

command -v jq >/dev/null 2>&1 || { echo >&2 "I require jq but it's not installed. Install with: apt-get install jq . (or on mac: brew install jq). Aborting."; exit 1; }

function cleanup_env {
    unset BUILD_NUMBER
}

function set_dst_s3_bucket_name_from_stack {
    S3_BUCKET_NAME=$(aws cloudformation describe-stacks --stack-name ${DST_BUCKET_STACK_NAME} | jq -r '.[]|.[].Outputs|.[]|select(.OutputKey=="name")|.OutputValue')
}

# ---------------------------------------
# CONFIG
# ---------------------------------------

[ ${PERSONAL_BUILD} == true ] && ./create_dev_stack.sh
set_templates_and_stack_names
[ ${PERSONAL_BUILD} == true ] && export BUILD_NUMBER=dev.$(date +%Y%m%d%H%M%S)

if [ ${cleanup} == true ]; then
    cleanup_env
    if [ ${PERSONAL_BUILD} == true ]; then
        (cd cfn/; cf delete -y ${FUNCTION_STACK_NAME}.yaml && rm ${FUNCTION_STACK_NAME}.yaml)
        set_dst_s3_bucket_name_from_stack
        aws s3 rm --recursive s3://${S3_BUCKET_NAME}/
        (cd cfn/; cf delete -y ${DST_BUCKET_STACK_NAME}.yaml && rm ${DST_BUCKET_STACK_NAME}.yaml)
        rm cfn/.gitignore
    fi
fi
# ---------------------------------------

if [ ${verbose} == true ]; then
    cat << EOF
executing deploy using:
DST_BUCKET_STACK_NAME:        ${DST_BUCKET_STACK_NAME}
FUNCTION_STACK_NAME:          ${FUNCTION_STACK_NAME}
PERSONAL_BUILD:               ${PERSONAL_BUILD}
BUILD_NUMBER:                 ${BUILD_NUMBER}
EOF
fi

echo "deploying INFRASTRUCTURE ..."
if [ ${DEPLOY_INFRASTRUCTURE} == true ]; then
  (cd cfn/; cf sync -y ${DST_BUCKET_STACK_NAME}.yaml)
else
  echo "SKIP deploying infrastructure!"
fi

function set_target_s3_key_for_lambda {
    local version=$(cat target/VERSION)
    local zipfile=$(ls target/ | grep -e '.zip$')
    S3_KEY_FOR_LAMBDA="dist/v${version}/${zipfile}"
}

echo "deploying CODE ..."
if [ ${DEPLOY_CODE} == true ]; then
    set_dst_s3_bucket_name_from_stack
    echo "deploying lambda zip to bucket: ${S3_BUCKET_NAME}"
    extra_opts=''
    [ ${verbose} == true ] && extra_opts='-X'
    pyb ${extra_opts} -P bucket_name="${S3_BUCKET_NAME}" -x verify upload_zip_to_s3

    echo "deploying/update lambda function ..."
    set_target_s3_key_for_lambda
    (cd cfn/; cf sync -y ${FUNCTION_STACK_NAME}.yaml -p ${FUNCTION_STACK_NAME}.s3key=${S3_KEY_FOR_LAMBDA})

    echo "integration tests ..."
    ./do_integration_tests.sh

else
   echo "SKIP deploying Code!"
fi

export FUNCTION_STACK_NAME=${FUNCTION_STACK_NAME}
export DST_BUCKET_STACK_NAME=${DST_BUCKET_STACK_NAME}
<commit_msg>bugfix env handling in deploy<commit_after>#!/usr/bin/env bash

#
# build, test and deploy
#

set -o errexit
set -o pipefail
set -o nounset
# set -o xtrace

source helper.sh

# Usage info
show_help() {
    cat << EOF
Usage: ${0##*/} [-hv] [-ci] 
    Build and deploy everything. You should specify what to deploy (code, infrastructure)
    Specify at least one of -c or -i.

    -c 		build and deploy lambda function. Can be used with -i too.
    -d          delete needed enviroment variables. cleanup. defaults are used.
    -h          display this help and exit
    -i          deploy infrastructure. Can be used with -c too.
    -p          disable personal build/deploy for development 
                normally cloudformation stacks will be suffixed with your local username (3) 
    -v          verbose mode.

    example usage:
    ./deploy.sh -ci
    ./deploy.sh -ci -p      to deploy to production (no personal build)
EOF
}

# A POSIX variable
OPTIND=1         # Reset in case getopts has been used previously in the shell.

# Initialize our own variables:
verbose=false
cleanup=false
DEPLOY_INFRASTRUCTURE=false
DEPLOY_CODE=false
PERSONAL_BUILD=true

while getopts "h?vicdp" opt; do
    case "$opt" in
    h|\?)
        show_help
        exit 0
        ;;
    v)  verbose=true
        ;;
    d)  cleanup=true
        ;;
    i)  DEPLOY_INFRASTRUCTURE=true
        ;;
    c)  DEPLOY_CODE=true
        ;;
    p)  PERSONAL_BUILD=false
        ;;
    esac
done

shift $((OPTIND-1))

if [ ${DEPLOY_INFRASTRUCTURE} == false ] && [ ${DEPLOY_CODE} == false ] && [ ${cleanup} == false ]; then
    show_help
    exit 0
fi

# check preconditions
command -v cf >/dev/null 2>&1 || { echo >&2 "I require cfn-sphere but it's not installed. Install with: pip install cfn-sphere. Aborting."; exit 1; }

command -v jq >/dev/null 2>&1 || { echo >&2 "I require jq but it's not installed. Install with: apt-get install jq . (or on mac: brew install jq). Aborting."; exit 1; }

function cleanup_env {
    unset BUILD_NUMBER
    unset DST_BUCKET_STACK_NAME
    unset FUNCTION_STACK_NAME
    unset PERSONAL_BUILD
}

function set_dst_s3_bucket_name_from_stack {
    S3_BUCKET_NAME=$(aws cloudformation describe-stacks --stack-name ${DST_BUCKET_STACK_NAME} | jq -r '.[]|.[].Outputs|.[]|select(.OutputKey=="name")|.OutputValue')
}

# ---------------------------------------
# CONFIG
# ---------------------------------------

[ ${PERSONAL_BUILD} == true ] && ./create_dev_stack.sh
set_templates_and_stack_names
[ ${PERSONAL_BUILD} == true ] && export BUILD_NUMBER=dev.$(date +%Y%m%d%H%M%S)

if [ ${cleanup} == true ]; then
    cleanup_env
    if [ ${PERSONAL_BUILD} == true ]; then
        (cd cfn/; cf delete -y ${FUNCTION_STACK_NAME}.yaml && rm ${FUNCTION_STACK_NAME}.yaml)
        set_dst_s3_bucket_name_from_stack
        aws s3 rm --recursive s3://${S3_BUCKET_NAME}/
        (cd cfn/; cf delete -y ${DST_BUCKET_STACK_NAME}.yaml && rm ${DST_BUCKET_STACK_NAME}.yaml)
        rm cfn/.gitignore
    fi
fi
# ---------------------------------------

if [ ${verbose} == true ]; then
    cat << EOF
executing deploy using:
DST_BUCKET_STACK_NAME:        ${DST_BUCKET_STACK_NAME}
FUNCTION_STACK_NAME:          ${FUNCTION_STACK_NAME}
PERSONAL_BUILD:               ${PERSONAL_BUILD}
BUILD_NUMBER:                 ${BUILD_NUMBER}
EOF
fi

export FUNCTION_STACK_NAME=${FUNCTION_STACK_NAME}
export DST_BUCKET_STACK_NAME=${DST_BUCKET_STACK_NAME}

echo "deploying INFRASTRUCTURE ..."
if [ ${DEPLOY_INFRASTRUCTURE} == true ]; then
  (cd cfn/; cf sync -y ${DST_BUCKET_STACK_NAME}.yaml)
else
  echo "SKIP deploying infrastructure!"
fi

function set_target_s3_key_for_lambda {
    local version=$(cat target/VERSION)
    local zipfile=$(ls target/ | grep -e '.zip$')
    S3_KEY_FOR_LAMBDA="dist/v${version}/${zipfile}"
}

echo "deploying CODE ..."
if [ ${DEPLOY_CODE} == true ]; then
    set_dst_s3_bucket_name_from_stack
    echo "deploying lambda zip to bucket: ${S3_BUCKET_NAME}"
    extra_opts=''
    [ ${verbose} == true ] && extra_opts='-X'
    pyb ${extra_opts} -P bucket_name="${S3_BUCKET_NAME}" -x verify upload_zip_to_s3

    echo "deploying/update lambda function ..."
    set_target_s3_key_for_lambda
    (cd cfn/; cf sync -y ${FUNCTION_STACK_NAME}.yaml -p ${FUNCTION_STACK_NAME}.s3key=${S3_KEY_FOR_LAMBDA})

    echo "integration tests ..."
    ./do_integration_tests.sh

else
   echo "SKIP deploying Code!"
fi

{'fraction_non_alphanumeric': 0.14440350069092583, 'fraction_numerical': 0.006678949792722248, 'mean_word_length': 2.7306701030927836, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1809347', 'n_tokens_mistral': 3060, 'n_tokens_neox': 2985, 'n_words': 803}