<commit_before>readonly CORE_CONFIG="$DATA_DIR/core.json"
readonly CORE_COMPONENTS="postgresql \
  vault \
  gitlab \
  swarm \
  rabbitmq"

###########################################################
export CORE_COMPONENTS_LIST=""
export CORE_MACHINES_LIST=""

validate_core_config() {
  # TODO: check if components.json has all the require components
  __process_msg "Validating core config"
  CORE_COMPONENTS_LIST=$(cat "$CORE_CONFIG" | jq '.')
  local component_count=$(echo $CORE_COMPONENTS_LIST | jq '. | length')
  if [[ $component_count -lt 1 ]]; then
    __process_msg "5 components required to set up Shippable, $component_count provided"
    exit 1
  else
    __process_msg "Component count: $component_count"
  fi

}

install_database() {
  __process_msg "Installing Database"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $db_host | jq '.ip')
  ##TODO:
  # - prommt user for db username and password
  # - copy the installation script to remote machine
  # - run sed command to replace username/password with user input
  # - once complete, save the values in satefile
  _copy_script_remote $host "installPostgresql.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/installPostgresql.sh"
}


save_db_credentials_in_statefile() {
  __process_msg "Saving database credentials in state file"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local db_ip=$(echo $db_host | jq -r '.ip')
  local db_port=5432
  local db_address=$db_ip":"$db_port

  db_name="shipdb"
  db_username="apiuser"
  db_password="<PASSWORD>"
  db_dialect="postgres"

  result=$(cat $STATE_FILE | jq '.systemSettings.dbHost = "'$db_ip'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbDialect = "'$db_dialect'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbPort = "'$db_port'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbname = "'$db_name'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbUsername = "'$db_username'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbPassword = "'$<PASSWORD>'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbUrl = "'$db_address'"')
  echo $result > $STATE_FILE
}

save_db_credentials() {
  __process_msg "Saving database credentials"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $db_host | jq '.ip')
  local db_ip=$(echo $db_host | jq '.ip')
  local db_port=5432
  local db_username=$(cat $STATE_FILE | jq '.systemSettings.dbUsername')
  local db_password=$(cat $STATE_FILE | jq '.systemSettings.dbPassword')
  local db_address=$db_ip:$db_port

  #TODO: fetch db_name from state.json
  local db_name="shipdb"

  _copy_script_remote $host ".pgpass" "/root/"
  _exec_remote_cmd $host "sed -i \"s/{{address}}/$db_address/g\" /root/.pgpass"
  _exec_remote_cmd $host "sed -i \"s/{{database}}/$db_name/g\" /root/.pgpass"
  _exec_remote_cmd $host "sed -i \"s/{{username}}/$db_username/g\" /root/.pgpass"
  _exec_remote_cmd $host "sed -i \"s/{{password}}/$db_password/g\" /root/.pgpass"

  _exec_remote_cmd $host "chmod 0600 /root/.pgpass"
}

run_migrations() {
  __process_msg "Please copy migrations.sql onto machine which runs database, type (y) when done"
  __process_msg "Done? (y/n)"
  read response
  if [[ "$response" =~ "y" ]]; then
    __process_msg "Proceeding with steps to run migrations"
    #TODO: Run migrations on db
  else
    __process_msg "Migrations are required to install core"
    run_migrations
  fi
}

install_vault() {
  __process_msg "Installing Vault"
  local vault_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $vault_host | jq '.ip')
  _copy_script_remote $host "installVault.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/installVault.sh"

  local vault_url=$host

  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local db_ip=$(echo $db_host | jq '.ip')
  local db_port=5432
  local db_username=$(cat $STATE_FILE | jq '.systemSettings.dbUsername')
  local db_address=$db_ip:$db_port

  #TODO: fetch db_name from state.json
  local db_name="shipdb"
  local VAULT_JSON_FILE="/vault/config/scripts/vaultConfig.json"

  _copy_script_remote $host "vault.hcl" "/etc/vault.d/"
  _copy_script_remote $host "policy.hcl" "/etc/vault.d/"
  _copy_script_remote $host "vault_kv_store.sql" "/etc/vault.d/"
  _copy_script_remote $host "vault.conf" "/etc/init/"
  _copy_script_remote $host "vaultConfig.json" "/etc/vault.d/"

  _exec_remote_cmd $host "sed -i \"s/{{DB_USERNAME}}/$db_username/g\" /etc/vault.d/vault.hcl"
  _exec_remote_cmd $host "sed -i \"s/{{DB_PASSWORD}}/$db_password/g\" /etc/vault.d/vault.hcl"
  _exec_remote_cmd $host "sed -i \"s/{{DB_ADDRESS}}/$db_address/g\" /etc/vault.d/vault.hcl"

  _exec_remote_cmd $host "psql -U $db_username -h $db_ip -d $db_name -w -f /etc/vault.d/vault_kv_store.sql"

  _exec_remote_cmd $host "sudo service vault start || true"

  _copy_script_remote $host "bootstrapVault.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/bootstrapVault.sh $db_username $db_name $db_ip $vault_url"

  _copy_script_local $host $VAULT_JSON_FILE
}

save_vault_credentials() {
  __process_msg "Saving vault credentials in state.json"
  local VAULT_FILE="/tmp/shippable/vaultConfig.json"

  local vault_url=$(cat $VAULT_FILE | jq '.vaultUrl')
  local vault_token=$(cat $VAULT_FILE | jq '.vaultToken')

  result=$(cat $STATE_FILE | jq -r '.systemSettings.vaultUrl = "'$vault_url'"')
  echo $result | jq '.' > $STATE_FILE

  result=$(cat $STATE_FILE | jq -r '.systemSettings.vaultToken = "'$vault_token'"')
  echo $result | jq '.' > $STATE_FILE
  __process_msg "Vault credentials successfully saved to state.json"
}

install_rabbitmq() {
  __process_msg "Installing RabbitMQ"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $db_host | jq -r '.ip')
  _copy_script_remote $host "installRabbit.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/installRabbit.sh"

  _copy_script_remote $host "rabbitmqadmin" "$SCRIPT_DIR_REMOTE"

  # TODO: The user should be prompted to enter a username and password, which should be
  # used by the bootstrapRabbit.sh
  _copy_script_remote $host "bootstrapRabbit.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/bootstrapRabbit.sh"

  local amqp_user="SHIPPABLETESTUSER"
  local amqp_pass="<PASSWORD>"
  local amqp_exchange="shippableEx"
  local amqp_port=5672
  local amqp_port_admin=15672

  local amqp_url="amqp://$amqp_user:$amqp_pass@$host:$amqp_port/shippable"
  local update=$(cat $STATE_FILE | jq '.systemSettings.amqpUrl = "'$amqp_url'"')
  echo $update | jq '.' > $STATE_FILE

  local amqp_url_root="amqp://$amqp_user:$amqp_pass@$host:$amqp_port/shippableRoot"
  update=$(cat $STATE_FILE | jq '.systemSettings.amqpUrlRoot = "'$amqp_url_root'"')
  echo $update | jq '.' > $STATE_FILE<commit_msg>more vault fixes<commit_after>readonly CORE_CONFIG="$DATA_DIR/core.json"
readonly CORE_COMPONENTS="postgresql \
  vault \
  gitlab \
  swarm \
  rabbitmq"

###########################################################
export CORE_COMPONENTS_LIST=""
export CORE_MACHINES_LIST=""

validate_core_config() {
  # TODO: check if components.json has all the require components
  __process_msg "Validating core config"
  CORE_COMPONENTS_LIST=$(cat "$CORE_CONFIG" | jq '.')
  local component_count=$(echo $CORE_COMPONENTS_LIST | jq '. | length')
  if [[ $component_count -lt 1 ]]; then
    __process_msg "5 components required to set up Shippable, $component_count provided"
    exit 1
  else
    __process_msg "Component count: $component_count"
  fi

}

install_database() {
  __process_msg "Installing Database"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $db_host | jq '.ip')
  ##TODO:
  # - prommt user for db username and password
  # - copy the installation script to remote machine
  # - run sed command to replace username/password with user input
  # - once complete, save the values in satefile
  _copy_script_remote $host "installPostgresql.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/installPostgresql.sh"
}


save_db_credentials_in_statefile() {
  __process_msg "Saving database credentials in state file"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local db_ip=$(echo $db_host | jq -r '.ip')
  local db_port=5432
  local db_address=$db_ip":"$db_port

  db_name="shipdb"
  db_username="apiuser"
  db_password="<PASSWORD>"
  db_dialect="postgres"

  result=$(cat $STATE_FILE | jq '.systemSettings.dbHost = "'$db_ip'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbDialect = "'$db_dialect'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbPort = "'$db_port'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbname = "'$db_name'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbUsername = "'$db_username'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbPassword = "'$<PASSWORD>'"')
  echo $result > $STATE_FILE

  result=$(cat $STATE_FILE | jq '.systemSettings.dbUrl = "'$db_address'"')
  echo $result > $STATE_FILE
}

save_db_credentials() {
  __process_msg "Saving database credentials"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $db_host | jq '.ip')
  local db_ip=$(echo $db_host | jq '.ip')
  local db_port=5432
  local db_username=$(cat $STATE_FILE | jq '.systemSettings.dbUsername')
  local db_password=$(cat $STATE_FILE | jq '.systemSettings.dbPassword')
  local db_address=$db_ip:$db_port

  #TODO: fetch db_name from state.json
  local db_name="shipdb"

  _copy_script_remote $host ".pgpass" "/root/"
  _exec_remote_cmd $host "sed -i \"s/{{address}}/$db_address/g\" /root/.pgpass"
  _exec_remote_cmd $host "sed -i \"s/{{database}}/$db_name/g\" /root/.pgpass"
  _exec_remote_cmd $host "sed -i \"s/{{username}}/$db_username/g\" /root/.pgpass"
  _exec_remote_cmd $host "sed -i \"s/{{password}}/$db_password/g\" /root/.pgpass"

  _exec_remote_cmd $host "chmod 0600 /root/.pgpass"
}

run_migrations() {
  __process_msg "Please copy migrations.sql onto machine which runs database, type (y) when done"
  __process_msg "Done? (y/n)"
  read response
  if [[ "$response" =~ "y" ]]; then
    __process_msg "Proceeding with steps to run migrations"
    #TODO: Run migrations on db
  else
    __process_msg "Migrations are required to install core"
    run_migrations
  fi
}

install_vault() {
  __process_msg "Installing Vault"
  local vault_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $vault_host | jq '.ip')
  _copy_script_remote $host "installVault.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/installVault.sh"

  local vault_url=$host

  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local db_ip=$(echo $db_host | jq '.ip')
  local db_port=5432
  local db_username=$(cat $STATE_FILE | jq '.systemSettings.dbUsername')
  local db_address=$db_ip:$db_port

  local db_name="shipdb"
  local VAULT_JSON_FILE="/etc/vault.d/vaultConfig.json"

  _copy_script_remote $host "vault.hcl" "/etc/vault.d/"
  _copy_script_remote $host "policy.hcl" "/etc/vault.d/"
  _copy_script_remote $host "vault_kv_store.sql" "/etc/vault.d/"
  _copy_script_remote $host "vault.conf" "/etc/init/"
  _copy_script_remote $host "vaultConfig.json" "/etc/vault.d/"

  _exec_remote_cmd $host "sed -i \"s/{{DB_USERNAME}}/$db_username/g\" /etc/vault.d/vault.hcl"
  _exec_remote_cmd $host "sed -i \"s/{{DB_PASSWORD}}/$db_password/g\" /etc/vault.d/vault.hcl"
  _exec_remote_cmd $host "sed -i \"s/{{DB_ADDRESS}}/$db_address/g\" /etc/vault.d/vault.hcl"

  _exec_remote_cmd $host "psql -U $db_username -h $db_ip -d $db_name -w -f /etc/vault.d/vault_kv_store.sql"

  _exec_remote_cmd $host "sudo service vault start || true"

  _copy_script_remote $host "bootstrapVault.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/bootstrapVault.sh $db_username $db_name $db_ip $vault_url"

  _copy_script_local $host $VAULT_JSON_FILE
}

save_vault_credentials() {
  __process_msg "Saving vault credentials in state.json"
  local VAULT_FILE="/tmp/shippable/vaultConfig.json"

  local vault_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $vault_host | jq -r '.ip')
  local vault_url="http://$vault_ip:8200"
  result=$(cat $STATE_FILE | jq -r '.systemSettings.vaultUrl = "'$vault_url'"')
  update=$(echo $result | jq '.' | tee $STATE_FILE)

  local vault_token=$(cat $VAULT_FILE | jq -r '.vaultToken')
  result=$(cat $STATE_FILE | jq -r '.systemSettings.vaultToken = "'$vault_token'"')
  update=$(echo $result | jq '.' | tee $STATE_FILE)
  __process_msg "Vault credentials successfully saved to state.json"
}

install_rabbitmq() {
  __process_msg "Installing RabbitMQ"
  local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
  local host=$(echo $db_host | jq -r '.ip')
  _copy_script_remote $host "installRabbit.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/installRabbit.sh"

  _copy_script_remote $host "rabbitmqadmin" "$SCRIPT_DIR_REMOTE"

  # TODO: The user should be prompted to enter a username and password, which should be
  # used by the bootstrapRabbit.sh
  _copy_script_remote $host "bootstrapRabbit.sh" "$SCRIPT_DIR_REMOTE"
  _exec_remote_cmd "$host" "$SCRIPT_DIR_REMOTE/bootstrapRabbit.sh"

  local amqp_user="SHIPPABLETESTUSER"
  local amqp_pass="<PASSWORD>"
  local amqp_exchange="shippableEx"
  local amqp_port=5672
  local amqp_port_admin=15672

  local amqp_url="amqp://$amqp_user:$amqp_pass@$host:$amqp_port/shippable"
  local update=$(cat $STATE_FILE | jq '.systemSettings.amqpUrl = "'$amqp_url'"')
  echo $update | jq '.' > $STATE_FILE

  local amqp_url_root="amqp://$amqp_user:$amqp_pass@$host:$amqp_port/shippableRoot"
  update=$(cat $STATE_FILE | jq '.systemSettings.amqpUrlRoot = "'$amqp_url_root'"')
  echo $update | jq '.' > $STATE_FILE
{'fraction_non_alphanumeric': 0.154522527435076, 'fraction_numerical': 0.0040897007702269785, 'mean_word_length': 4.906602254428341, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269717', 'n_tokens_mistral': 5564, 'n_tokens_neox': 5331, 'n_words': 1187}