<commit_before>#!/bin/bash

# Copyright 2014 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A library of helper functions and constant for the local config.

# Use the config file specified in $KUBE_CONFIG_FILE, or default to
# config-default.sh.

KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/azure/${KUBE_CONFIG_FILE-"config-default.sh"}"
source "${KUBE_ROOT}/cluster/common.sh"

function azure_call {
    local -a params=()
    local param
    # the '... in "$@"' is implicit on a for, so doesn't need to be stated.
    for param; do
        params+=("${param}")
    done
    local rc=0
    local stderr
    local count=0
    while [[ count -lt 10 ]]; do
        stderr=$(azure "${params[@]}" 2>&1 >&3) && break
        rc=$?
        if [[ "${stderr}" != *"getaddrinfo ENOTFOUND"* ]]; then
            break
        fi
        count=$(($count + 1))
    done 3>&1
    if [[ "${rc}" -ne 0 ]]; then
        echo "${stderr}" >&2
        return "${rc}"
    fi
}

function json_val () {
    python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1'';
}

# Verify prereqs
function verify-prereqs {
    if [[ -z "$(which azure)" ]]; then
        echo "Couldn't find azure in PATH"
        echo "  please install with 'npm install azure-cli'"
        exit 1
    fi

    if [[ -z "$(azure_call account list | grep true)" ]]; then
        echo "Default azure account not set"
        echo "  please set with 'azure account set'"
        exit 1
    fi

    account=$(azure_call account list | grep true)
    if which md5 > /dev/null 2>&1; then
        AZ_HSH=$(md5 -q -s "$account")
    else
        AZ_HSH=$(echo -n "$account" | md5sum)
    fi

    AZ_HSH=${AZ_HSH:0:7}
    AZ_STG=kube$AZ_HSH
    echo "==> AZ_STG: $AZ_STG"

    AZ_CS="$AZ_CS_PREFIX-$AZ_HSH"
    echo "==> AZ_CS: $AZ_CS"

    CONTAINER=kube-$TAG
    echo "==> CONTAINER: $CONTAINER"
}

# Create a temp dir that'll be deleted at the end of this bash session.
#
# Vars set:
#   KUBE_TEMP
function ensure-temp-dir {
    if [[ -z ${KUBE_TEMP-} ]]; then
        KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
        trap 'rm -rf "${KUBE_TEMP}"' EXIT
    fi
}

# Verify and find the various tar files that we are going to use on the server.
#
# Vars set:
#   SERVER_BINARY_TAR
#   SALT_TAR
function find-release-tars {
    SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz"
    if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
        SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz"
    fi
    if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
        echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz"
        exit 1
    fi

    SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz"
    if [[ ! -f "$SALT_TAR" ]]; then
        SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz"
    fi
    if [[ ! -f "$SALT_TAR" ]]; then
        echo "!!! Cannot find kubernetes-salt.tar.gz"
        exit 1
    fi
}


# Take the local tar files and upload them to Azure Storage.  They will then be
# downloaded by the master as part of the start up script for the master.
#
# Assumed vars:
#   SERVER_BINARY_TAR
#   SALT_TAR
# Vars set:
#   SERVER_BINARY_TAR_URL
#   SALT_TAR_URL
function upload-server-tars() {
    SERVER_BINARY_TAR_URL=
    SALT_TAR_URL=

    echo "==> SERVER_BINARY_TAR: $SERVER_BINARY_TAR"
    echo "==> SALT_TAR: $SALT_TAR"

    echo "+++ Staging server tars to Azure Storage: $AZ_STG"
    local server_binary_url="${SERVER_BINARY_TAR##*/}"
    local salt_url="${SALT_TAR##*/}"

    SERVER_BINARY_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$server_binary_url"
    SALT_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$salt_url"

    echo "==> SERVER_BINARY_TAR_URL: $SERVER_BINARY_TAR_URL"
    echo "==> SALT_TAR_URL: $SALT_TAR_URL"

    echo "--> Checking storage exists..."
    if [[ -z "$(azure_call storage account show $AZ_STG 2>/dev/null | \
    grep data)" ]]; then
        echo "--> Creating storage..."
        azure_call storage account create -l "$AZ_LOCATION" $AZ_STG
    fi

    echo "--> Getting storage key..."
    stg_key=$(azure_call storage account keys list $AZ_STG --json | \
        json_val '["primaryKey"]')

    echo "--> Checking storage container exists..."
    if [[ -z "$(azure_call storage container show -a $AZ_STG -k "$stg_key" \
      $CONTAINER 2>/dev/null | grep data)" ]]; then
        echo "--> Creating storage container..."
        azure_call storage container create \
            -a $AZ_STG \
            -k "$stg_key" \
            -p Blob \
            $CONTAINER
    fi

    echo "--> Checking server binary exists in the container..."
    if [[ -n "$(azure_call storage blob show -a $AZ_STG -k "$stg_key" \
      $CONTAINER $server_binary_url 2>/dev/null | grep data)" ]]; then
        echo "--> Deleting server binary in the container..."
        azure_call storage blob delete \<commit_msg>Stop prompting for storage account type<commit_after>#!/bin/bash

# Copyright 2014 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A library of helper functions and constant for the local config.

# Use the config file specified in $KUBE_CONFIG_FILE, or default to
# config-default.sh.

KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/azure/${KUBE_CONFIG_FILE-"config-default.sh"}"
source "${KUBE_ROOT}/cluster/common.sh"

function azure_call {
    local -a params=()
    local param
    # the '... in "$@"' is implicit on a for, so doesn't need to be stated.
    for param; do
        params+=("${param}")
    done
    local rc=0
    local stderr
    local count=0
    while [[ count -lt 10 ]]; do
        stderr=$(azure "${params[@]}" 2>&1 >&3) && break
        rc=$?
        if [[ "${stderr}" != *"getaddrinfo ENOTFOUND"* ]]; then
            break
        fi
        count=$(($count + 1))
    done 3>&1
    if [[ "${rc}" -ne 0 ]]; then
        echo "${stderr}" >&2
        return "${rc}"
    fi
}

function json_val () {
    python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1'';
}

# Verify prereqs
function verify-prereqs {
    if [[ -z "$(which azure)" ]]; then
        echo "Couldn't find azure in PATH"
        echo "  please install with 'npm install azure-cli'"
        exit 1
    fi

    if [[ -z "$(azure_call account list | grep true)" ]]; then
        echo "Default azure account not set"
        echo "  please set with 'azure account set'"
        exit 1
    fi

    account=$(azure_call account list | grep true)
    if which md5 > /dev/null 2>&1; then
        AZ_HSH=$(md5 -q -s "$account")
    else
        AZ_HSH=$(echo -n "$account" | md5sum)
    fi

    AZ_HSH=${AZ_HSH:0:7}
    AZ_STG=kube$AZ_HSH
    echo "==> AZ_STG: $AZ_STG"

    AZ_CS="$AZ_CS_PREFIX-$AZ_HSH"
    echo "==> AZ_CS: $AZ_CS"

    CONTAINER=kube-$TAG
    echo "==> CONTAINER: $CONTAINER"
}

# Create a temp dir that'll be deleted at the end of this bash session.
#
# Vars set:
#   KUBE_TEMP
function ensure-temp-dir {
    if [[ -z ${KUBE_TEMP-} ]]; then
        KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
        trap 'rm -rf "${KUBE_TEMP}"' EXIT
    fi
}

# Verify and find the various tar files that we are going to use on the server.
#
# Vars set:
#   SERVER_BINARY_TAR
#   SALT_TAR
function find-release-tars {
    SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz"
    if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
        SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz"
    fi
    if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
        echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz"
        exit 1
    fi

    SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz"
    if [[ ! -f "$SALT_TAR" ]]; then
        SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz"
    fi
    if [[ ! -f "$SALT_TAR" ]]; then
        echo "!!! Cannot find kubernetes-salt.tar.gz"
        exit 1
    fi
}


# Take the local tar files and upload them to Azure Storage.  They will then be
# downloaded by the master as part of the start up script for the master.
#
# Assumed vars:
#   SERVER_BINARY_TAR
#   SALT_TAR
# Vars set:
#   SERVER_BINARY_TAR_URL
#   SALT_TAR_URL
function upload-server-tars() {
    SERVER_BINARY_TAR_URL=
    SALT_TAR_URL=

    echo "==> SERVER_BINARY_TAR: $SERVER_BINARY_TAR"
    echo "==> SALT_TAR: $SALT_TAR"

    echo "+++ Staging server tars to Azure Storage: $AZ_STG"
    local server_binary_url="${SERVER_BINARY_TAR##*/}"
    local salt_url="${SALT_TAR##*/}"

    SERVER_BINARY_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$server_binary_url"
    SALT_TAR_URL="https://${AZ_STG}.blob.core.windows.net/$CONTAINER/$salt_url"

    echo "==> SERVER_BINARY_TAR_URL: $SERVER_BINARY_TAR_URL"
    echo "==> SALT_TAR_URL: $SALT_TAR_URL"

    echo "--> Checking storage exists..."
    if [[ -z "$(azure_call storage account show $AZ_STG 2>/dev/null | \
    grep data)" ]]; then
        echo "--> Creating storage..."
        azure_call storage account create -l "$AZ_LOCATION" $AZ_STG --type LRS
    fi

    echo "--> Getting storage key..."
    stg_key=$(azure_call storage account keys list $AZ_STG --json | \
        json_val '["primaryKey"]')

    echo "--> Checking storage container exists..."
    if [[ -z "$(azure_call storage container show -a $AZ_STG -k "$stg_key" \
      $CONTAINER 2>/dev/null | grep data)" ]]; then
        echo "--> Creating storage container..."
        azure_call storage container create \
            -a $AZ_STG \
            -k "$stg_key" \
            -p Blob \
            $CONTAINER
    fi

    echo "--> Checking server binary exists in the container..."
    if [[ -n "$(azure_call storage blob show -a $AZ_STG -k "$stg_key" \
      $CONTAINER $server_binary_url 2>/dev/null | grep data)" ]]; then
        echo "--> Deleting server binary in the container..."
        azure_call storage blob delete \
{'fraction_non_alphanumeric': 0.1358305300481862, 'fraction_numerical': 0.007455223202109283, 'mean_word_length': 2.751705320600273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 49, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5285504', 'n_tokens_mistral': 3971, 'n_tokens_neox': 3732, 'n_words': 1085}