<commit_before>#!/bin/bash -e

export skip_step=0
export sleep_time=1

_update_install_status() {
  local update=$(cat $STATE_FILE | jq '.installStatus.'"$1"'='true'')
  _update_state "$update"
}

_check_component_status() {
  local status=$(cat $STATE_FILE | jq '.installStatus.'"$1"'')
  if [ "$status" = true ]; then
    skip_step=1;
  fi
}

generate_serviceuser_token() {
  skip_step=0
  _check_component_status "serviceuserTokenGenerated"
  if [ $skip_step -eq 0 ]; then
    __process_msg "Generating random token for serviceuser"
    local token=$(cat /proc/sys/kernel/random/uuid)
    local stateToken=$(cat $STATE_FILE | jq '.systemSettings.serviceUserToken="'$token'"')
    echo $stateToken > $STATE_FILE
    _update_install_status "serviceuserTokenGenerated"
  else
    __process_msg "Service user token already generated, skipping"
  fi
}

update_docker_creds() {
  skip_step=0
  _check_component_status "dockerCredsUpdated"
  if [ $skip_step -eq 0 ]; then
    __process_msg "Updating docker credentials to pull shippable images"
    local gitlab_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="swarm")')
    local host=$(echo "$gitlab_host" | jq '.ip')

    local credentials_template="$REMOTE_SCRIPTS_DIR/credentials.template"
    local credentials_file="$REMOTE_SCRIPTS_DIR/credentials"

    __process_msg "Updating : installerAccessKey"
    local aws_access_key=$(cat $STATE_FILE | jq -r '.systemSettings.installerAccessKey')
    sed "s#{{aws_access_key}}#$aws_access_key#g" $credentials_template > $credentials_file

    __process_msg "Updating : installerSecretKey"
    local aws_secret_key=$(cat $STATE_FILE | jq -r '.systemSettings.installerSecretKey')
    sed -i "s#{{aws_secret_key}}#$aws_secret_key#g" $credentials_file

    _copy_script_remote $host "credentials" "/root/.aws/"
    local save_docker_login_cmd='aws ecr --region us-east-1 get-login > /tmp/docker_login.sh'
    _exec_remote_cmd $host "$save_docker_login_cmd"
    local update_perm='chmod +x /tmp/docker_login.sh'
    _exec_remote_cmd $host "$update_perm"
    local docker_login_cmd='/tmp/docker_login.sh'
    _exec_remote_cmd $host "$docker_login_cmd"

    _update_install_status "dockerCredsUpdated"
  else
    __process_msg "Docker credentials already updated, skipping"
  fi
}

generate_system_config() {
  skip_step=0
  _check_component_status "systemConfigSqlCreated"
  if [ $skip_step -eq 0 ]; then
    __process_msg "Inserting data into systemConfigs Table"
    local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
    local host=$(echo $db_host | jq '.ip')
    local db_ip=$(echo $db_host | jq '.ip')

    #TODO: put sed update into a function and call it for each variable
    local system_configs_template="$REMOTE_SCRIPTS_DIR/systemConfigsData.sql.template"
    local system_configs_sql="$REMOTE_SCRIPTS_DIR/system_configs_data.sql"

    # NOTE:
    # "sed" is using '#' as a separator in following statements
    __process_msg "Updating : defaultMinionCount"
    local default_minion_count=$(cat $STATE_FILE | jq -r '.systemSettings.defaultMinionCount')
    sed "s#{{DEFAULT_MINION_COUNT}}#$default_minion_count#g" $system_configs_template > $system_configs_sql

    __process_msg "Updating : defaultPipelineCount"
    local default_pipeline_count=$(cat $STATE_FILE | jq -r '.systemSettings.defaultPipelineCount')
    sed -i "s#{{DEFAULT_PIPELINE_COUNT}}#$default_pipeline_count#g" $system_configs_sql

    __process_msg "Updating : serverEnabled"
    local server_enabled=$(cat $STATE_FILE | jq -r '.systemSettings.serverEnabled')
    sed -i "s#{{SERVER_ENABLED}}#$server_enabled#g" $system_configs_sql

    __process_msg "Updating : buildTimeout"
    local build_timeout=$(cat $STATE_FILE | jq -r '.systemSettings.buildTimeoutMS')
    sed -i "s#{{BUILD_TIMEOUT_MS}}#$build_timeout#g" $system_configs_sql

    __process_msg "Updating : defaultPrivateJobQuota"
    local private_job_quota=$(cat $STATE_FILE | jq -r '.systemSettings.defaultPrivateJobQuota')
    sed -i "s#{{DEFAULT_PRIVATE_JOB_QUOTA}}#$private_job_quota#g" $system_configs_sql

    __process_msg "Updating : serviceuserToken"
    local serviceuser_token=$(cat $STATE_FILE | jq -r '.systemSettings.serviceUserToken')
    sed -i "s#{{SERVICE_USER_TOKEN}}#$serviceuser_token#g" $system_configs_sql

    __process_msg "Updating : vaultUrl"
    local vault_url=$(cat $STATE_FILE | jq -r '.systemSettings.vaultUrl')
    sed -i "s#{{VAULT_URL}}#$vault_url#g" $system_configs_sql

    __process_msg "Updating : vaultToken"
    local vault_token=$(cat $STATE_FILE | jq -r '.systemSettings.vaultToken')
    sed -i "s#{{VAULT_TOKEN}}#$vault_token#g" $system_configs_sql

    __process_msg "Updating : vaultRefreshTime"
    local vault_refresh_time=$(cat $STATE_FILE | jq -r '.systemSettings.vaultRefreshTimeInSec')
    sed -i "s#{{VAULT_REFRESH_TIME_SEC}}#$vault_refresh_time#g" $system_configs_sql

    __process_msg "Updating : cachingEnabled"
    local caching_enabled=$(cat $STATE_FILE | jq -r '.systemSettings.cachingEnabled')
    sed -i "s#{{CACHING_ENABLED}}#$caching_enabled#g" $system_configs_sql

    __process_msg "Updating : hubspotEnabled"
    local hubspot_enabled=$(cat $STATE_FILE | jq -r '.systemSettings.hubspotEnabled')
    sed -i "s#{{HUBSPOT_ENABLED}}#$hubspot_enabled#g" $system_configs_sql

    __process_msg "Updating : amqpUrl"
    local amqp_url=$(cat $STATE_FILE | jq -r '.systemSettings.amqpUrl')
    sed -i "s#{{AMQP_URL}}#$amqp_url#g" $system_configs_sql

    __process_msg "Updating : amqpUrlAdmin"
    local amqp_url_admin=$(cat $STATE_FILE | jq -r '.systemSettings.amqpUrlAdmin')
    sed -i "s#{{AMQP_URL_ADMIN}}#$amqp_url_admin#g" $system_configs_sql

    __process_msg "Updating : amqpUrlRoot"
    local amqp_url_root=$(cat $STATE_FILE | jq -r '.systemSettings.amqpUrlRoot')
    sed -i "s#{{AMQP_URL_ROOT}}#$amqp_url_root#g" $system_configs_sql

    __process_msg "Updating : amqpDefaultExchange"
    local amqp_default_exchange=$(cat $STATE_FILE | jq -r '.systemSettings.amqpDefaultExchange')
    sed -i "s#{{AMQP_DEFAULT_EXCHANGE}}#$amqp_default_exchange#g" $system_configs_sql

    __process_msg "Updating : apiUrl"
    local api_url=$(cat $STATE_FILE | jq -r '.systemSettings.apiUrl')
    sed -i "s#{{API_URL}}#$api_url#g" $system_configs_sql

    __process_msg "Updating : apiPort"
    local api_port=$(cat $STATE_FILE | jq -r '.systemSettings.apiPort')
    sed -i "s#{{API_PORT}}#$api_port#g" $system_configs_sql

    __process_msg "Updating : wwwUrl"
    local www_url=$(cat $STATE_FILE | jq -r '.systemSettings.wwwUrl')
    sed -i "s#{{WWW_URL}}#$www_url#g" $system_configs_sql

    __process_msg "Updating : runMode"
    local run_mode=$(cat $STATE_FILE | jq -r '.systemSettings.runMode')
    sed -i "s#{{RUN_MODE}}#$run_mode#g" $system_configs_sql

    __process_msg "Updating : rootQueueList"
    local root_queue_list=$(cat $STATE_FILE | jq -r '.systemSettings.rootQueueList')
    sed -i "s#{{ROOT_QUEUE_LIST}}#$root_queue_list#g" $system_configs_sql

    __process_msg "Updating : execImage"
    local exec_image=$(cat $STATE_FILE | jq -r '.systemSettings.execImage')
    sed -i "s#{{EXEC_IMAGE}}#$exec_image#g" $system_configs_sql

    __process_msg "Updating : createdAt"
    local created_at=$(date)
    sed -i "s#{{CREATED_AT}}#$created_at#g" $system_configs_sql

    __process_msg "Updating : updatedAt"
    sed -i "s#{{UPDATED_AT}}#$created_at#g" $system_configs_sql

    __process_msg "Updating : dynamicNodesSystemIntegrationId"
    local dynamic_nodes_system_integration_id=$(cat $STATE_FILE | jq -r '.systemSettings.dynamicNodesSystemIntegrationId')
    sed -i "s#{{DYNAMIC_NODES_SYSTEM_INTEGRATION_ID}}#$dynamic_nodes_system_integration_id#g" $system_configs_sql

    __process_msg "Updating : systemNodePrivateKey"
    local system_node_private_key=$(cat $STATE_FILE | jq -r '.systemSettings.systemNodePrivateKey')<commit_msg>Fixed error when inserting ssh private key.<commit_after>#!/bin/bash -e

export skip_step=0
export sleep_time=1

_update_install_status() {
  local update=$(cat $STATE_FILE | jq '.installStatus.'"$1"'='true'')
  _update_state "$update"
}

_check_component_status() {
  local status=$(cat $STATE_FILE | jq '.installStatus.'"$1"'')
  if [ "$status" = true ]; then
    skip_step=1;
  fi
}

generate_serviceuser_token() {
  skip_step=0
  _check_component_status "serviceuserTokenGenerated"
  if [ $skip_step -eq 0 ]; then
    __process_msg "Generating random token for serviceuser"
    local token=$(cat /proc/sys/kernel/random/uuid)
    local stateToken=$(cat $STATE_FILE | jq '.systemSettings.serviceUserToken="'$token'"')
    echo $stateToken > $STATE_FILE
    _update_install_status "serviceuserTokenGenerated"
  else
    __process_msg "Service user token already generated, skipping"
  fi
}

update_docker_creds() {
  skip_step=0
  _check_component_status "dockerCredsUpdated"
  if [ $skip_step -eq 0 ]; then
    __process_msg "Updating docker credentials to pull shippable images"
    local gitlab_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="swarm")')
    local host=$(echo "$gitlab_host" | jq '.ip')

    local credentials_template="$REMOTE_SCRIPTS_DIR/credentials.template"
    local credentials_file="$REMOTE_SCRIPTS_DIR/credentials"

    __process_msg "Updating : installerAccessKey"
    local aws_access_key=$(cat $STATE_FILE | jq -r '.systemSettings.installerAccessKey')
    sed "s#{{aws_access_key}}#$aws_access_key#g" $credentials_template > $credentials_file

    __process_msg "Updating : installerSecretKey"
    local aws_secret_key=$(cat $STATE_FILE | jq -r '.systemSettings.installerSecretKey')
    sed -i "s#{{aws_secret_key}}#$aws_secret_key#g" $credentials_file

    _copy_script_remote $host "credentials" "/root/.aws/"
    local save_docker_login_cmd='aws ecr --region us-east-1 get-login > /tmp/docker_login.sh'
    _exec_remote_cmd $host "$save_docker_login_cmd"
    local update_perm='chmod +x /tmp/docker_login.sh'
    _exec_remote_cmd $host "$update_perm"
    local docker_login_cmd='/tmp/docker_login.sh'
    _exec_remote_cmd $host "$docker_login_cmd"

    _update_install_status "dockerCredsUpdated"
  else
    __process_msg "Docker credentials already updated, skipping"
  fi
}

generate_system_config() {
  skip_step=0
  _check_component_status "systemConfigSqlCreated"
  if [ $skip_step -eq 0 ]; then
    __process_msg "Inserting data into systemConfigs Table"
    local db_host=$(cat $STATE_FILE | jq '.machines[] | select (.group=="core" and .name=="db")')
    local host=$(echo $db_host | jq '.ip')
    local db_ip=$(echo $db_host | jq '.ip')

    #TODO: put sed update into a function and call it for each variable
    local system_configs_template="$REMOTE_SCRIPTS_DIR/systemConfigsData.sql.template"
    local system_configs_sql="$REMOTE_SCRIPTS_DIR/system_configs_data.sql"

    # NOTE:
    # "sed" is using '#' as a separator in following statements
    __process_msg "Updating : defaultMinionCount"
    local default_minion_count=$(cat $STATE_FILE | jq -r '.systemSettings.defaultMinionCount')
    sed "s#{{DEFAULT_MINION_COUNT}}#$default_minion_count#g" $system_configs_template > $system_configs_sql

    __process_msg "Updating : defaultPipelineCount"
    local default_pipeline_count=$(cat $STATE_FILE | jq -r '.systemSettings.defaultPipelineCount')
    sed -i "s#{{DEFAULT_PIPELINE_COUNT}}#$default_pipeline_count#g" $system_configs_sql

    __process_msg "Updating : serverEnabled"
    local server_enabled=$(cat $STATE_FILE | jq -r '.systemSettings.serverEnabled')
    sed -i "s#{{SERVER_ENABLED}}#$server_enabled#g" $system_configs_sql

    __process_msg "Updating : buildTimeout"
    local build_timeout=$(cat $STATE_FILE | jq -r '.systemSettings.buildTimeoutMS')
    sed -i "s#{{BUILD_TIMEOUT_MS}}#$build_timeout#g" $system_configs_sql

    __process_msg "Updating : defaultPrivateJobQuota"
    local private_job_quota=$(cat $STATE_FILE | jq -r '.systemSettings.defaultPrivateJobQuota')
    sed -i "s#{{DEFAULT_PRIVATE_JOB_QUOTA}}#$private_job_quota#g" $system_configs_sql

    __process_msg "Updating : serviceuserToken"
    local serviceuser_token=$(cat $STATE_FILE | jq -r '.systemSettings.serviceUserToken')
    sed -i "s#{{SERVICE_USER_TOKEN}}#$serviceuser_token#g" $system_configs_sql

    __process_msg "Updating : vaultUrl"
    local vault_url=$(cat $STATE_FILE | jq -r '.systemSettings.vaultUrl')
    sed -i "s#{{VAULT_URL}}#$vault_url#g" $system_configs_sql

    __process_msg "Updating : vaultToken"
    local vault_token=$(cat $STATE_FILE | jq -r '.systemSettings.vaultToken')
    sed -i "s#{{VAULT_TOKEN}}#$vault_token#g" $system_configs_sql

    __process_msg "Updating : vaultRefreshTime"
    local vault_refresh_time=$(cat $STATE_FILE | jq -r '.systemSettings.vaultRefreshTimeInSec')
    sed -i "s#{{VAULT_REFRESH_TIME_SEC}}#$vault_refresh_time#g" $system_configs_sql

    __process_msg "Updating : cachingEnabled"
    local caching_enabled=$(cat $STATE_FILE | jq -r '.systemSettings.cachingEnabled')
    sed -i "s#{{CACHING_ENABLED}}#$caching_enabled#g" $system_configs_sql

    __process_msg "Updating : hubspotEnabled"
    local hubspot_enabled=$(cat $STATE_FILE | jq -r '.systemSettings.hubspotEnabled')
    sed -i "s#{{HUBSPOT_ENABLED}}#$hubspot_enabled#g" $system_configs_sql

    __process_msg "Updating : amqpUrl"
    local amqp_url=$(cat $STATE_FILE | jq -r '.systemSettings.amqpUrl')
    sed -i "s#{{AMQP_URL}}#$amqp_url#g" $system_configs_sql

    __process_msg "Updating : amqpUrlAdmin"
    local amqp_url_admin=$(cat $STATE_FILE | jq -r '.systemSettings.amqpUrlAdmin')
    sed -i "s#{{AMQP_URL_ADMIN}}#$amqp_url_admin#g" $system_configs_sql

    __process_msg "Updating : amqpUrlRoot"
    local amqp_url_root=$(cat $STATE_FILE | jq -r '.systemSettings.amqpUrlRoot')
    sed -i "s#{{AMQP_URL_ROOT}}#$amqp_url_root#g" $system_configs_sql

    __process_msg "Updating : amqpDefaultExchange"
    local amqp_default_exchange=$(cat $STATE_FILE | jq -r '.systemSettings.amqpDefaultExchange')
    sed -i "s#{{AMQP_DEFAULT_EXCHANGE}}#$amqp_default_exchange#g" $system_configs_sql

    __process_msg "Updating : apiUrl"
    local api_url=$(cat $STATE_FILE | jq -r '.systemSettings.apiUrl')
    sed -i "s#{{API_URL}}#$api_url#g" $system_configs_sql

    __process_msg "Updating : apiPort"
    local api_port=$(cat $STATE_FILE | jq -r '.systemSettings.apiPort')
    sed -i "s#{{API_PORT}}#$api_port#g" $system_configs_sql

    __process_msg "Updating : wwwUrl"
    local www_url=$(cat $STATE_FILE | jq -r '.systemSettings.wwwUrl')
    sed -i "s#{{WWW_URL}}#$www_url#g" $system_configs_sql

    __process_msg "Updating : runMode"
    local run_mode=$(cat $STATE_FILE | jq -r '.systemSettings.runMode')
    sed -i "s#{{RUN_MODE}}#$run_mode#g" $system_configs_sql

    __process_msg "Updating : rootQueueList"
    local root_queue_list=$(cat $STATE_FILE | jq -r '.systemSettings.rootQueueList')
    sed -i "s#{{ROOT_QUEUE_LIST}}#$root_queue_list#g" $system_configs_sql

    __process_msg "Updating : execImage"
    local exec_image=$(cat $STATE_FILE | jq -r '.systemSettings.execImage')
    sed -i "s#{{EXEC_IMAGE}}#$exec_image#g" $system_configs_sql

    __process_msg "Updating : createdAt"
    local created_at=$(date)
    sed -i "s#{{CREATED_AT}}#$created_at#g" $system_configs_sql

    __process_msg "Updating : updatedAt"
    sed -i "s#{{UPDATED_AT}}#$created_at#g" $system_configs_sql

    __process_msg "Updating : dynamicNodesSystemIntegrationId"
    local dynamic_nodes_system_integration_id=$(cat $STATE_FILE | jq -r '.systemSettings.dynamicNodesSystemIntegrationId')
    sed -i "s#{{DYNAMIC_NODES_SYSTEM_INTEGRATION_ID}}#$dynamic_nodes_system_integration_id#g" $system_configs_sql

    __process_msg "Updating : systemNodePrivateKey"
    local system_node_private_key=$(cat $STATE_FILE | jq '.systemSettings.systemNodePrivateKey' | sed s/\"//g)
{'fraction_non_alphanumeric': 0.12303813425780019, 'fraction_numerical': 0.0015127639457926253, 'mean_word_length': 5.175943947061113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4191793', 'n_tokens_mistral': 5643, 'n_tokens_neox': 5524, 'n_words': 1070}