<commit_before>#!/bin/bash

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# <http://www.apache.org/licenses/LICENSE-2.0>
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

print_usage() {
    cat << EOF
usage: sackmesser deploy -e ENV -o ORG [--googleapi | --apigeeapi] [-t TOKEN | -u USER -p PASSWORD] [options]

Apigee deployment utility.

Options:
--googleapi (default), use apigee.googleapi.com (for X, hybrid)
--apigeeapi, use api.enterprise.apigee.com (for Edge)
-b,--base-path, overrides the default base path for the API proxy
-d,--directory, path to the apiproxy or shared flow bundle to be deployed
-e,--environment, Apigee environment name
-g,--github, Link to proxy or shared flow bundle on github
-n,--name, Overrides the default API proxy or shared flow name
-o,--organization, Apigee organization name
-u,--username, Apigee User Name (Edge only)
-p,--password, Apigee User Password (Edge only)
-m,--mfa, Apigee MFA code (Edge only)
-t,--token, GCP token (X,hybrid only) or OAuth2 token (Edge)
-h,--hostname, publicly reachable hostname for the environment
--description, Human friendly proxy or proxy description
EOF
}

if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
    print_usage
    exit 0
fi

if ! which xmllint > /dev/null; then
    echo "[FATAL] please install xmllint command"
    exit 1
fi

while [ "$#" -gt 0 ]; do
  case "$1" in
    -b) base_path="$2"; shift 2;;
    -d) directory="$2"; shift 2;;
    -e) environment="$2"; shift 2;;
    -g) url="$2"; shift 2;;
    -m) mfa="$2"; shift 2;;
    -n) bundle_name="$2"; shift 2;;
    -o) organization="$2"; shift 2;;
    -p) password="$2"; shift 2;;
    -t) token="$2"; shift 2;;
    -u) username="$2"; shift 2;;
    -h) hostname="$2"; shift 2;;

    --directory) directory="${1}"; shift 2;;
    --github) url="${1}"; shift 2;;
    --token) token="${1}"; shift 2;;
    --mfa) mfa="${1}"; shift 2;;
    --name) bundle_name="${1}"; shift 2;;
    --base-path) base-path="${1}"; shift 2;;
    --username) username="${1}"; shift 2;;
    --password) password="${1}"; shift 2;;
    --environment) environment="${1}"; shift 2;;
    --organization) organization="${1}"; shift 2;;
    --hostname) hostname="${1}"; shift 2;;
    --description) description="${2}"; shift 2;;

    --apigeeapi) apiversion="apigee"; shift 1;;
    --googleapi) apiversion="google"; shift 1;;

    -*) echo "[FATAL] unknown option: $1" >&2; exit 1;;
    *) echo "[FATAL] unknown positional argument $1"; exit 1;;
  esac
done

if [[ -z "$token" && (-z "$username"  || -z "$password") ]]; then
    echo "[FATAL] required either -t (OAuth2 or GCP access token) or -u and -p (Edge username and password)"
    exit 1
fi

if [[ -z "$apiversion" ]]; then
    echo "[INFO] using default API version: Google API (apigee.googleapi.com)"
    echo "[INFO] for Apigee Edge (api.enterprise.apigee.com) please specify --apigeeapi"
    apiversion="google"
fi

# Make temp 'deploy' directory to keep things clean
temp_folder="$PWD/deploy-$(date +%s)-$RANDOM"
rm -rf "$temp_folder" && mkdir -p "$temp_folder"
cleanup() {
  echo "[INFO] removing $temp_folder"
  rm  -rf "$temp_folder"
}
trap cleanup EXIT

SCRIPT_FOLDER=$( (cd "$(dirname "$0")" && pwd ))

# copy resources to temp directory
if [ -n "$url" ]; then
    pattern='https?:\/\/github.com\/([^\/]*)\/([^\/]*)\/tree\/([^\/]*)\/(.*)'
    [[ "$url" =~ $pattern ]]
    git_org="${BASH_REMATCH[1]}"
    git_repo="${BASH_REMATCH[2]}"
    git_branch="${BASH_REMATCH[3]}"
    git_path="${BASH_REMATCH[4]}"

    git clone "https://github.com/${git_org}/${git_repo}.git" "$temp_folder/$git_repo"
    (cd "$temp_folder/$git_repo" && git checkout "$git_branch")
    cp -R "$temp_folder/$git_repo/$git_path/"* "$temp_folder"
    ls -la "$temp_folder"
else
    source_dir="${directory:-$PWD}"
    echo "[INFO] using local directory: $source_dir"
    [ -d "$source_dir/apiproxy" ] && cp -r "$source_dir/apiproxy" "$temp_folder/apiproxy"
    [ -d "$source_dir/sharedflowbundle" ] && cp -r "$source_dir/sharedflowbundle" "$temp_folder/sharedflowbundle"
    [ -e "$source_dir/edge.json" ] && cp "$source_dir/edge.json" "$temp_folder/"
fi

# Config Deployment
if [ -f "$temp_folder"/edge.json ]; then
    echo "[INFO] Preparing config $temp_folder/edge.json"

    config_action='update'

    if [ "$apiversion" = "google" ]; then

        jq --arg APIGEE_ENV "$environment" -c '.envConfig[$APIGEE_ENV].kvms[]? | .' < "$temp_folder"/edge.json | while read -r line; do
            kvm=$(echo "$line" | jq -c 'del(.entry)')
            kvmname=$(echo "$kvm" | jq -r '.name')
            echo "[INFO] X/hybrid patch: adding kvm: $kvmname"
            curl -X POST --fail "https://apigee.googleapis.com/v1/organizations/$organization/environments/$environment/keyvaluemaps" \
                -H "Authorization: Bearer $token" \
                -H "Content-Type: application/json" \
                --data "$kvm" || echo "assuming the kvm already exists"

            KVM_ADMIN_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" "https://apigee.googleapis.com/v1/organizations/$organization/environments/$environment/apis/kvm-admin-v1/deployments")

            echo "[INFO] kvm admin status $KVM_ADMIN_STATUS"

            if [ "$KVM_ADMIN_STATUS" != "200" ];then
                "$SCRIPT_FOLDER/deploy.sh" --googleapi -d "$SCRIPT_FOLDER/../../../../references/kvm-admin-api" -t "$token" \
                    -o "$organization" -e "$environment"
            fi

            echo "$line" | jq -c  '.entry[]? | .' | while read -r kvmentry; do
                kvmentry=$(echo "$kvmentry" | jq '.["key"]=.name | del(.name)')
                echo "[INFO] adding entry: $(echo "$kvmentry" | jq '.key')"
                curl -s -X POST -k --fail "https://$hostname/kvm-admin/v1/organizations/$organization/environments/$environment/keyvaluemaps/$kvmname/entries" \
                    -H "Authorization: Bearer $token" \
                    -H "Content-Type: application/json" \
                    --data "$kvmentry" > /dev/null
            done
        done
    fi
fi


if [ -d "$temp_folder/apiproxy" ]; then
    echo "[INFO] Configuring API Proxy"

    # Determine Proxy name
    name_in_bundle="$(xmllint --xpath 'string(//APIProxy/@name)' "$temp_folder"/apiproxy/*.xml)"
    bundle_name=${bundle_name:=$name_in_bundle}

    # (optional) Override base path
    if [ -n "$base_path" ]; then
        echo "[INFO] Setting base path: $base_path"
        sed -i.bak "s|<BasePath>.*</BasePath>|<BasePath>$base_path<\/BasePath>|g" "$temp_folder"/apiproxy/proxies/*.xml
        rm "$temp_folder"/apiproxy/proxies/*.xml.bak
    fi

    # (optional) Set Proxy Description
    if [ -n "$description" ]; then
        echo "[INFO] Setting description: $description"
        sed -i.bak "s|^.*<Description>.*</Description>||g" "$temp_folder"/apiproxy/*.xml
        sed -i.bak "s|</APIProxy>|  <Description>$description</Description>\\n</APIProxy>|g" "$temp_folder"/apiproxy/*.xml
        rm "$temp_folder"/apiproxy/*.xml.bak
    fi<commit_msg>better KVM admin proxy debug in Sackmesser<commit_after>#!/bin/bash

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# <http://www.apache.org/licenses/LICENSE-2.0>
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

print_usage() {
    cat << EOF
usage: sackmesser deploy -e ENV -o ORG [--googleapi | --apigeeapi] [-t TOKEN | -u USER -p PASSWORD] [options]

Apigee deployment utility.

Options:
--googleapi (default), use apigee.googleapi.com (for X, hybrid)
--apigeeapi, use api.enterprise.apigee.com (for Edge)
-b,--base-path, overrides the default base path for the API proxy
-d,--directory, path to the apiproxy or shared flow bundle to be deployed
-e,--environment, Apigee environment name
-g,--github, Link to proxy or shared flow bundle on github
-n,--name, Overrides the default API proxy or shared flow name
-o,--organization, Apigee organization name
-u,--username, Apigee User Name (Edge only)
-p,--password, Apigee User Password (Edge only)
-m,--mfa, Apigee MFA code (Edge only)
-t,--token, GCP token (X,hybrid only) or OAuth2 token (Edge)
-h,--hostname, publicly reachable hostname for the environment
--description, Human friendly proxy or proxy description
EOF
}

if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
    print_usage
    exit 0
fi

if ! which xmllint > /dev/null; then
    echo "[FATAL] please install xmllint command"
    exit 1
fi

while [ "$#" -gt 0 ]; do
  case "$1" in
    -b) base_path="$2"; shift 2;;
    -d) directory="$2"; shift 2;;
    -e) environment="$2"; shift 2;;
    -g) url="$2"; shift 2;;
    -m) mfa="$2"; shift 2;;
    -n) bundle_name="$2"; shift 2;;
    -o) organization="$2"; shift 2;;
    -p) password="$2"; shift 2;;
    -t) token="$2"; shift 2;;
    -u) username="$2"; shift 2;;
    -h) hostname="$2"; shift 2;;

    --directory) directory="${1}"; shift 2;;
    --github) url="${1}"; shift 2;;
    --token) token="${1}"; shift 2;;
    --mfa) mfa="${1}"; shift 2;;
    --name) bundle_name="${1}"; shift 2;;
    --base-path) base-path="${1}"; shift 2;;
    --username) username="${1}"; shift 2;;
    --password) password="${1}"; shift 2;;
    --environment) environment="${1}"; shift 2;;
    --organization) organization="${1}"; shift 2;;
    --hostname) hostname="${1}"; shift 2;;
    --description) description="${2}"; shift 2;;

    --apigeeapi) apiversion="apigee"; shift 1;;
    --googleapi) apiversion="google"; shift 1;;

    -*) echo "[FATAL] unknown option: $1" >&2; exit 1;;
    *) echo "[FATAL] unknown positional argument $1"; exit 1;;
  esac
done

if [[ -z "$token" && (-z "$username"  || -z "$password") ]]; then
    echo "[FATAL] required either -t (OAuth2 or GCP access token) or -u and -p (Edge username and password)"
    exit 1
fi

if [[ -z "$apiversion" ]]; then
    echo "[INFO] using default API version: Google API (apigee.googleapi.com)"
    echo "[INFO] for Apigee Edge (api.enterprise.apigee.com) please specify --apigeeapi"
    apiversion="google"
fi

# Make temp 'deploy' directory to keep things clean
temp_folder="$PWD/deploy-$(date +%s)-$RANDOM"
rm -rf "$temp_folder" && mkdir -p "$temp_folder"
cleanup() {
  echo "[INFO] removing $temp_folder"
  rm  -rf "$temp_folder"
}
trap cleanup EXIT

SCRIPT_FOLDER=$( (cd "$(dirname "$0")" && pwd ))

# copy resources to temp directory
if [ -n "$url" ]; then
    pattern='https?:\/\/github.com\/([^\/]*)\/([^\/]*)\/tree\/([^\/]*)\/(.*)'
    [[ "$url" =~ $pattern ]]
    git_org="${BASH_REMATCH[1]}"
    git_repo="${BASH_REMATCH[2]}"
    git_branch="${BASH_REMATCH[3]}"
    git_path="${BASH_REMATCH[4]}"

    git clone "https://github.com/${git_org}/${git_repo}.git" "$temp_folder/$git_repo"
    (cd "$temp_folder/$git_repo" && git checkout "$git_branch")
    cp -R "$temp_folder/$git_repo/$git_path/"* "$temp_folder"
    ls -la "$temp_folder"
else
    source_dir="${directory:-$PWD}"
    echo "[INFO] using local directory: $source_dir"
    [ -d "$source_dir/apiproxy" ] && cp -r "$source_dir/apiproxy" "$temp_folder/apiproxy"
    [ -d "$source_dir/sharedflowbundle" ] && cp -r "$source_dir/sharedflowbundle" "$temp_folder/sharedflowbundle"
    [ -e "$source_dir/edge.json" ] && cp "$source_dir/edge.json" "$temp_folder/"
fi

# Config Deployment
if [ -f "$temp_folder"/edge.json ]; then
    echo "[INFO] Preparing config $temp_folder/edge.json"

    config_action='update'

    if [ "$apiversion" = "google" ]; then

        jq --arg APIGEE_ENV "$environment" -c '.envConfig[$APIGEE_ENV].kvms[]? | .' < "$temp_folder"/edge.json | while read -r line; do
            kvm=$(echo "$line" | jq -c 'del(.entry)')
            kvmname=$(echo "$kvm" | jq -r '.name')
            echo "[INFO] X/hybrid patch: adding kvm: $kvmname"
            curl -X POST --fail "https://apigee.googleapis.com/v1/organizations/$organization/environments/$environment/keyvaluemaps" \
                -H "Authorization: Bearer $token" \
                -H "Content-Type: application/json" \
                --data "$kvm" || echo "assuming the kvm already exists"

            KVM_ADMIN_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" "https://apigee.googleapis.com/v1/organizations/$organization/environments/$environment/apis/kvm-admin-v1/deployments")

            echo "[INFO] kvm admin status $KVM_ADMIN_STATUS"

            if [ "$KVM_ADMIN_STATUS" != "200" ];then
                "$SCRIPT_FOLDER/deploy.sh" --googleapi -d "$SCRIPT_FOLDER/../../../../references/kvm-admin-api" -t "$token" \
                    -o "$organization" -e "$environment"
            fi

            echo "$line" | jq -c  '.entry[]? | .' | while read -r kvmentry; do
                kvmentry=$(echo "$kvmentry" | jq '.["key"]=.name | del(.name)')
                echo "[INFO] adding entry: $(echo "$kvmentry" | jq '.key')"
                curl -s -X POST -k --fail "https://$hostname/kvm-admin/v1/organizations/$organization/environments/$environment/keyvaluemaps/$kvmname/entries" \
                    -H "Authorization: Bearer $token" \
                    -H "Content-Type: application/json" \
                    --data "$kvmentry" > /dev/null || ( echo "[FATAL] failed to add entry to KVM: https://$hostname/kvm-admin/v1/organizations/$organization/environments/$environment/keyvaluemaps/$kvmname" && exit 1 )
            done
        done
    fi
fi


if [ -d "$temp_folder/apiproxy" ]; then
    echo "[INFO] Configuring API Proxy"

    # Determine Proxy name
    name_in_bundle="$(xmllint --xpath 'string(//APIProxy/@name)' "$temp_folder"/apiproxy/*.xml)"
    bundle_name=${bundle_name:=$name_in_bundle}

    # (optional) Override base path
    if [ -n "$base_path" ]; then
        echo "[INFO] Setting base path: $base_path"
        sed -i.bak "s|<BasePath>.*</BasePath>|<BasePath>$base_path<\/BasePath>|g" "$temp_folder"/apiproxy/proxies/*.xml
        rm "$temp_folder"/apiproxy/proxies/*.xml.bak
    fi

    # (optional) Set Proxy Description
    if [ -n "$description" ]; then
        echo "[INFO] Setting description: $description"
        sed -i.bak "s|^.*<Description>.*</Description>||g" "$temp_folder"/apiproxy/*.xml
        sed -i.bak "s|</APIProxy>|  <Description>$description</Description>\\n</APIProxy>|g" "$temp_folder"/apiproxy/*.xml
        rm "$temp_folder"/apiproxy/*.xml.bak
    fi
{'fraction_non_alphanumeric': 0.17704331699562856, 'fraction_numerical': 0.010994833752814943, 'mean_word_length': 3.027473993064817, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 9, 'lorem ipsum': 0, 'www.': 2, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4033083', 'n_tokens_mistral': 5287, 'n_tokens_neox': 5026, 'n_words': 1432}