<commit_before>#!/bin/bash
# This script provides commands to initialize and manage Kubeflow
# deployments.
#
# Basic usage
#
# kfctl.sh init myapp --platform generatic
# cd myapp
# kfctl.sh generate all
# kfctl.sh apply all
set -xe

COMMAND=$1
WHAT=$2

ENV_FILE="env.sh"

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source "${DIR}/util.sh"
source "${DIR}/gke/util.sh"
source "${DIR}/util-minikube.sh"

createEnv() {
  # Check if there is a file env.sh
  # If there is source it otherwise create it.
  # this ensures all relevant environment variables are persisted in
  # a file for consistency across runs.
  echo PLATFORM=${PLATFORM} >> ${ENV_FILE}
  DEFAULT_KUBEFLOW_REPO="$( cd "${DIR}/.." >/dev/null && pwd )"
  echo KUBEFLOW_REPO=${KUBEFLOW_REPO:-"${DEFAULT_KUBEFLOW_REPO}"} >> ${ENV_FILE}
  echo KUBEFLOW_VERSION=${KUBEFLOW_VERSION:-"master"} >> ${ENV_FILE}
  echo KUBEFLOW_KS_DIR=${KUBEFLOW_KS_DIR:-"$(pwd)/ks_app"} >> ${ENV_FILE}
  echo KUBEFLOW_DOCKER_REGISTRY=${KUBEFLOW_DOCKER_REGISTRY:-""} >> ${ENV_FILE}

  # Namespace where kubeflow is deployed
  echo K8S_NAMESPACE=${K8S_NAMESPACE:-"kubeflow"} >> ${ENV_FILE}

  case "$PLATFORM" in 
    minikube)
      echo KUBEFLOW_CLOUD=minikube >> ${ENV_FILE}
      echo MOUNT_LOCAL=${MOUNT_LOCAL} >> ${ENV_FILE}
      ;;
    ack)
      echo KUBEFLOW_CLOUD=ack >> ${ENV_FILE}
      echo KUBEFLOW_DOCKER_REGISTRY=registry.aliyuncs.com >> ${ENV_FILE}
      ;;
    gcp)
      PROJECT=${PROJECT:-$(gcloud config get-value project 2>/dev/null)}
      echo KUBEFLOW_CLOUD=gke >> ${ENV_FILE}
      echo PROJECT="${PROJECT}" >> ${ENV_FILE}
      if [ -z "${PROJECT}" ]; then
        echo PROJECT must be set either using environment variable PROJECT
        echo or by setting the default project in gcloud
        exit 1
      fi    
  
      # Name of the deployment
      DEPLOYMENT_NAME=${DEPLOYMENT_NAME:-"kubeflow"}
      echo DEPLOYMENT_NAME="${DEPLOYMENT_NAME}" >> ${ENV_FILE}
  
      # Kubeflow directories
      echo KUBEFLOW_DM_DIR=${KUBEFLOW_DM_DIR:-"$(pwd)/gcp_config"} >> ${ENV_FILE}
      echo KUBEFLOW_SECRETS_DIR=${KUBEFLOW_SECRETS_DIR:-"$(pwd)/secrets"} >> ${ENV_FILE}
      echo KUBEFLOW_K8S_MANIFESTS_DIR="$(pwd)/k8s_specs" >> ${ENV_FILE}
  
      # Name of the K8s context to create.
      echo  KUBEFLOW_K8S_CONTEXT=${DEPLOYMENT_NAME} >> ${ENV_FILE}
  
      # GCP Zone
      # The default should be a zone that supports Haswell.
      ZONE=${ZONE:-$(gcloud config get-value compute/zone 2>/dev/null)}
      echo ZONE=${ZONE:-"us-east1-d"} >> ${ENV_FILE}
  
      # Email for cert manager
      EMAIL=${EMAIL:-$(gcloud config get-value account 2>/dev/null)}
      echo EMAIL=${EMAIL} >> ${ENV_FILE}
  
      # GCP Static IP Name
      echo KUBEFLOW_IP_NAME=${KUBEFLOW_IP_NAME:-"${DEPLOYMENT_NAME}-ip"} >> ${ENV_FILE}
      # Name of the endpoint
      KUBEFLOW_ENDPOINT_NAME=${KUBEFLOW_ENDPOINT_NAME:-"${DEPLOYMENT_NAME}"}
      echo KUBEFLOW_ENDPOINT_NAME=${KUBEFLOW_ENDPOINT_NAME} >> ${ENV_FILE}
      # Complete hostname
      echo KUBEFLOW_HOSTNAME=${KUBEFLOW_HOSTNAME:-"${KUBEFLOW_ENDPOINT_NAME}.endpoints.${PROJECT}.cloud.goog"} >> ${ENV_FILE}
  
      echo CONFIG_FILE=${CONFIG_FILE:-"cluster-kubeflow.yaml"} >> ${ENV_FILE}
  
      if [ -z "${PROJECT_NUMBER}" ]; then
        PROJECT_NUMBER=$(gcloud projects describe ${PROJECT} --format='value(project_number)')
      fi
  
      echo PROJECT_NUMBER=${PROJECT_NUMBER} >> ${ENV_FILE}
      ;;
    *)
      echo KUBEFLOW_CLOUD=null >> ${ENV_FILE}
      ;;
  esac
}

if [ "${COMMAND}" == "init" ]; then
	DEPLOYMENT_NAME=${WHAT}

	while [ "$1" != "" ]; do
    case $1 in
    	--platform)                  shift
                                     PLATFORM=$1
                                     ;;
        --project)                   shift
                                     PROJECT=$1
                                     ;;
		--skipInitProject)           shift
                                     SKIP_INIT_PROJECT=$1
                                     ;;
        --email)                     shift
                                     EMAIL=$1
                                     ;;
      esac
      shift
	done

	mkdir -p ${DEPLOYMENT_NAME}
	# Most commands expect to be executed from the app directory
	cd ${DEPLOYMENT_NAME}
	createEnv
	source ${ENV_FILE}
	# TODO(jlewi): Should we default to directory name?
	# TODO(jlewi): This doesn't work if user doesn't provide name we will end up
	# interpreting parameters as the name. To fix this we need to check name doesn't start with --
	if [ -z "${DEPLOYMENT_NAME}" ]; then
  		echo "name must be provided"
  		echo "usage: kfctl init <name>"
  		exit 1
	fi
    if [ -d ${DEPLOYMENT_NAME} ]; then
		echo Directory ${DEPLOYMENT_NAME} already exists
		exit 1
	fi

	if [ -z "${PLATFORM}" ]; then
  		echo "--platform must be provided"
  		echo "usage: kfctl init <PLATFORM>"
  		exit 1
	fi
	source "${ENV_FILE}"

	# TODO(jlewi): How can we skip GCP project setup? Add a command line argument
	# to skip it?
	if [ "${PLATFORM}" == "gcp" ]; then
	  if ! ${SKIP_INIT_PROJECT}; then
	  	gcpInitProject
	  fi
	fi

fi

source ${ENV_FILE}

if [ -z "${COMMAND}" ]; then
  echo COMMAND must be provided
  usage
  exit 1
fi

if [ -z "${WHAT}" ]; then
  echo WHAT must be provided
  usage
  exit 1
fi

# TODO(ankushagarwal): verify ks version is higher than 0.11.0<commit_msg>Fix kfctl.sh gcpInitProject is always skipped (#1425)<commit_after>#!/bin/bash
# This script provides commands to initialize and manage Kubeflow
# deployments.
#
# Basic usage
#
# kfctl.sh init myapp --platform generatic
# cd myapp
# kfctl.sh generate all
# kfctl.sh apply all
set -xe

COMMAND=$1
WHAT=$2

ENV_FILE="env.sh"

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
source "${DIR}/util.sh"
source "${DIR}/gke/util.sh"
source "${DIR}/util-minikube.sh"

createEnv() {
  # Check if there is a file env.sh
  # If there is source it otherwise create it.
  # this ensures all relevant environment variables are persisted in
  # a file for consistency across runs.
  echo PLATFORM=${PLATFORM} >> ${ENV_FILE}
  DEFAULT_KUBEFLOW_REPO="$( cd "${DIR}/.." >/dev/null && pwd )"
  echo KUBEFLOW_REPO=${KUBEFLOW_REPO:-"${DEFAULT_KUBEFLOW_REPO}"} >> ${ENV_FILE}
  echo KUBEFLOW_VERSION=${KUBEFLOW_VERSION:-"master"} >> ${ENV_FILE}
  echo KUBEFLOW_KS_DIR=${KUBEFLOW_KS_DIR:-"$(pwd)/ks_app"} >> ${ENV_FILE}
  echo KUBEFLOW_DOCKER_REGISTRY=${KUBEFLOW_DOCKER_REGISTRY:-""} >> ${ENV_FILE}

  # Namespace where kubeflow is deployed
  echo K8S_NAMESPACE=${K8S_NAMESPACE:-"kubeflow"} >> ${ENV_FILE}

  case "$PLATFORM" in 
    minikube)
      echo KUBEFLOW_CLOUD=minikube >> ${ENV_FILE}
      echo MOUNT_LOCAL=${MOUNT_LOCAL} >> ${ENV_FILE}
      ;;
    ack)
      echo KUBEFLOW_CLOUD=ack >> ${ENV_FILE}
      echo KUBEFLOW_DOCKER_REGISTRY=registry.aliyuncs.com >> ${ENV_FILE}
      ;;
    gcp)
      PROJECT=${PROJECT:-$(gcloud config get-value project 2>/dev/null)}
      echo KUBEFLOW_CLOUD=gke >> ${ENV_FILE}
      echo PROJECT="${PROJECT}" >> ${ENV_FILE}
      if [ -z "${PROJECT}" ]; then
        echo PROJECT must be set either using environment variable PROJECT
        echo or by setting the default project in gcloud
        exit 1
      fi    
  
      # Name of the deployment
      DEPLOYMENT_NAME=${DEPLOYMENT_NAME:-"kubeflow"}
      echo DEPLOYMENT_NAME="${DEPLOYMENT_NAME}" >> ${ENV_FILE}
  
      # Kubeflow directories
      echo KUBEFLOW_DM_DIR=${KUBEFLOW_DM_DIR:-"$(pwd)/gcp_config"} >> ${ENV_FILE}
      echo KUBEFLOW_SECRETS_DIR=${KUBEFLOW_SECRETS_DIR:-"$(pwd)/secrets"} >> ${ENV_FILE}
      echo KUBEFLOW_K8S_MANIFESTS_DIR="$(pwd)/k8s_specs" >> ${ENV_FILE}
  
      # Name of the K8s context to create.
      echo  KUBEFLOW_K8S_CONTEXT=${DEPLOYMENT_NAME} >> ${ENV_FILE}
  
      # GCP Zone
      # The default should be a zone that supports Haswell.
      ZONE=${ZONE:-$(gcloud config get-value compute/zone 2>/dev/null)}
      echo ZONE=${ZONE:-"us-east1-d"} >> ${ENV_FILE}
  
      # Email for cert manager
      EMAIL=${EMAIL:-$(gcloud config get-value account 2>/dev/null)}
      echo EMAIL=${EMAIL} >> ${ENV_FILE}
  
      # GCP Static IP Name
      echo KUBEFLOW_IP_NAME=${KUBEFLOW_IP_NAME:-"${DEPLOYMENT_NAME}-ip"} >> ${ENV_FILE}
      # Name of the endpoint
      KUBEFLOW_ENDPOINT_NAME=${KUBEFLOW_ENDPOINT_NAME:-"${DEPLOYMENT_NAME}"}
      echo KUBEFLOW_ENDPOINT_NAME=${KUBEFLOW_ENDPOINT_NAME} >> ${ENV_FILE}
      # Complete hostname
      echo KUBEFLOW_HOSTNAME=${KUBEFLOW_HOSTNAME:-"${KUBEFLOW_ENDPOINT_NAME}.endpoints.${PROJECT}.cloud.goog"} >> ${ENV_FILE}
  
      echo CONFIG_FILE=${CONFIG_FILE:-"cluster-kubeflow.yaml"} >> ${ENV_FILE}
  
      if [ -z "${PROJECT_NUMBER}" ]; then
        PROJECT_NUMBER=$(gcloud projects describe ${PROJECT} --format='value(project_number)')
      fi
  
      echo PROJECT_NUMBER=${PROJECT_NUMBER} >> ${ENV_FILE}
      ;;
    *)
      echo KUBEFLOW_CLOUD=null >> ${ENV_FILE}
      ;;
  esac
}

if [ "${COMMAND}" == "init" ]; then
	DEPLOYMENT_NAME=${WHAT}

	while [ "$1" != "" ]; do
    case $1 in
    	--platform)                  shift
                                     PLATFORM=$1
                                     ;;
        --project)                   shift
                                     PROJECT=$1
                                     ;;
		--skipInitProject)           shift
                                     SKIP_INIT_PROJECT=$1
                                     ;;
        --email)                     shift
                                     EMAIL=$1
                                     ;;
      esac
      shift
	done

	mkdir -p ${DEPLOYMENT_NAME}
	# Most commands expect to be executed from the app directory
	cd ${DEPLOYMENT_NAME}
	createEnv
	source ${ENV_FILE}
	# TODO(jlewi): Should we default to directory name?
	# TODO(jlewi): This doesn't work if user doesn't provide name we will end up
	# interpreting parameters as the name. To fix this we need to check name doesn't start with --
	if [ -z "${DEPLOYMENT_NAME}" ]; then
  		echo "name must be provided"
  		echo "usage: kfctl init <name>"
  		exit 1
	fi
    if [ -d ${DEPLOYMENT_NAME} ]; then
		echo Directory ${DEPLOYMENT_NAME} already exists
		exit 1
	fi

	if [ -z "${PLATFORM}" ]; then
  		echo "--platform must be provided"
  		echo "usage: kfctl init <PLATFORM>"
  		exit 1
	fi
	source "${ENV_FILE}"

	# TODO(jlewi): How can we skip GCP project setup? Add a command line argument
	# to skip it?
	if [ "${PLATFORM}" == "gcp" ]; then
	  if [ ! ${SKIP_INIT_PROJECT} ]; then
	  	gcpInitProject
	  fi
	fi

fi

source ${ENV_FILE}

if [ -z "${COMMAND}" ]; then
  echo COMMAND must be provided
  usage
  exit 1
fi

if [ -z "${WHAT}" ]; then
  echo WHAT must be provided
  usage
  exit 1
fi

# TODO(ankushagarwal): verify ks version is higher than 0.11.0
{'fraction_non_alphanumeric': 0.13112017804154302, 'fraction_numerical': 0.005749258160237389, 'mean_word_length': 2.6142761394101877, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3169349', 'n_tokens_mistral': 3863, 'n_tokens_neox': 3757, 'n_words': 901}