<commit_before>#!/bin/bash -e
# $WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR
# $WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_REGION
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_ACCESS_KEY
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_SECRET_KEY
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_LOCATION
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_ENV
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_URL
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_USERNAME
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_PASSWORD
# $WERCKER_DEPLOY_SOFE_SERVICE_SOFE_SERVICE_NAME
# $WERCKER_DEPLOY_SOFE_SERVICE_DEBUG


DSS_VERSION=$(ls "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/")
mkdir -p ~/.aws/
# Create access key file so we dont mess up any env_vars
echo -e "[default]\nregion=$WERCKER_DEPLOY_SOFE_SERVICE_S3_REGION\naws_access_key_id = $WERCKER_DEPLOY_SOFE_SERVICE_S3_ACCESS_KEY\naws_secret_access_key = $WERCKER_DEPLOY_SOFE_SERVICE_S3_SECRET_KEY\n" > ~/.aws/config


if [ "$WERCKER_DEPLOY_SOFE_SERVICE_DEBUG" == 'true' ]
then
  head "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/$DSS_VERSION/$WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE"
fi

if [ "$(file --mime-type -b /pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/$DSS_VERSION/$WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE)" != 'application/x-gzip' ]
then
# gzip the files in place
find "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/" -type f -exec gzip "{}" \; -exec echo "{}" \; -exec mv "{}.gz" "{}" \;
else
echo "already gziped!"
fi

# Upload all the files
aws s3 sync "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR" "s3://$WERCKER_DEPLOY_SOFE_SERVICE_S3_LOCATION" --content-encoding gzip --cache-control "public, max-age=31556926"

# To get curl to output the response contents but store the http status in a variable, you have to create a file descriptor and redirect curl output to it.
# See http://superuser.com/questions/272265/getting-curl-to-output-http-status-code
exec 3>&1

# Deploy using the deplanifester
STATUSCODE=$(curl -w '%{http_code}' -o >(cat >&3) -d "{ \"service\":\"$WERCKER_DEPLOY_SOFE_SERVICE_SOFE_SERVICE_NAME\",\"url\":\"https://$WERCKER_DEPLOY_SOFE_SERVICE_S3_LOCATION/$DSS_VERSION/$WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE\" }" -X PATCH "$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_URL/services?env=$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_ENV" -H "Accept: application/json" -k -H "Content-Type: application/json" -u "$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_USERNAME:$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_PASSWORD")

if test $STATUSCODE -ne 200; then
	echo # New line
	echo "Failed to deploy sofe service. Deplanifester returned http status ${STATUSCODE}"
	exit 1
fi
<commit_msg>More output<commit_after>#!/bin/bash -e
# $WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR
# $WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_REGION
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_ACCESS_KEY
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_SECRET_KEY
# $WERCKER_DEPLOY_SOFE_SERVICE_S3_LOCATION
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_ENV
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_URL
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_USERNAME
# $WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_PASSWORD
# $WERCKER_DEPLOY_SOFE_SERVICE_SOFE_SERVICE_NAME
# $WERCKER_DEPLOY_SOFE_SERVICE_DEBUG


DSS_VERSION=$(ls "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/")
mkdir -p ~/.aws/
# Create access key file so we dont mess up any env_vars
echo -e "[default]\nregion=$WERCKER_DEPLOY_SOFE_SERVICE_S3_REGION\naws_access_key_id = $WERCKER_DEPLOY_SOFE_SERVICE_S3_ACCESS_KEY\naws_secret_access_key = $WERCKER_DEPLOY_SOFE_SERVICE_S3_SECRET_KEY\n" > ~/.aws/config


if [ "$WERCKER_DEPLOY_SOFE_SERVICE_DEBUG" == 'true' ]
then
  head "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/$DSS_VERSION/$WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE"
fi

if [ "$(file --mime-type -b /pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/$DSS_VERSION/$WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE)" != 'application/x-gzip' ]
then
# gzip the files in place
find "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR/" -type f -exec gzip "{}" \; -exec echo "{}" \; -exec mv "{}.gz" "{}" \;
else
echo "already gziped!"
fi

# Upload all the files
aws s3 sync "/pipeline/source/$WERCKER_DEPLOY_SOFE_SERVICE_UPLOAD_DIR" "s3://$WERCKER_DEPLOY_SOFE_SERVICE_S3_LOCATION" --content-encoding gzip --cache-control "public, max-age=31556926"

# To get curl to output the response contents but store the http status in a variable, you have to create a file descriptor and redirect curl output to it.
# See http://superuser.com/questions/272265/getting-curl-to-output-http-status-code
exec 3>&1

# Deploy using the deplanifester
STATUSCODE=$(curl -w '%{http_code}' -o >(cat >&3) -d "{ \"service\":\"$WERCKER_DEPLOY_SOFE_SERVICE_SOFE_SERVICE_NAME\",\"url\":\"https://$WERCKER_DEPLOY_SOFE_SERVICE_S3_LOCATION/$DSS_VERSION/$WERCKER_DEPLOY_SOFE_SERVICE_MAIN_FILE\" }" -X PATCH "$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_URL/services?env=$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_ENV" -H "Accept: application/json" -k -H "Content-Type: application/json" -u "$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_USERNAME:$WERCKER_DEPLOY_SOFE_SERVICE_DEPLANIFESTER_PASSWORD")

echo # New line
echo "Deplanifester status code was ${STATUSCODE}"
echo # New line

if test $STATUSCODE -ne 200;
then
	echo "Failed to deploy sofe service. Deplanifester returned http status ${STATUSCODE}"
	exit 1
else
	echo "Successful deployment of sofe service"
fi

{'fraction_non_alphanumeric': 0.10349854227405247, 'fraction_numerical': 0.011661807580174927, 'mean_word_length': 7.496904024767802, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147733', 'n_tokens_mistral': 2316, 'n_tokens_neox': 2363, 'n_words': 347}