<commit_before>name: DataRepo Alpha Tests and Prod GCR Promotion
env:
  K8_CLUSTER: jade-master-us-central1
  GOOGLE_APPLICATION_CREDENTIALS: /tmp/alpha-test-runner.json
  GOOGLE_CLOUD_PROJECT: terra-datarepo-alpha
  GOOGLE_CLOUD_DATA_PROJECT: terra-datarepo-alpha-data
  TDR_LOG_APPENDER: Console-Standard
  GCR_DEV_URL: gcr.io/broad-jade-dev
  GCR_PUBLIC_URL: gcr.io/datarepo-public-gcr
  API_REPO: jade-data-repo
  UI_REPO: jade-data-repo-ui
on:
  # Jenkins will kick this job off manually after Alpha is deployed
  workflow_dispatch: {}
jobs:
  alpha-tests-and-gcr-promotion:
    runs-on: ubuntu-latest
    steps:
    - name: "Checkout jade-data-repo and latest tags"
      uses: actions/checkout@v2
      with:
        fetch-depth: 0
    - name: "Fetch current alpha version from /configuration endpoint"
      id: configuration
      run: |
        alpha_version_api=$(curl -s -X GET "https://data.alpha.envs-terra.bio/configuration" -H "accept: application/json" | jq -r '.semVer|rtrimstr("-SNAPSHOT")')
        echo "::set-output name=alpha_version_api::$alpha_version_api"
        echo "Alpha Version: $alpha_version_api"
    - name: "Checkout tag for DataBiosphere/jade-data-repo"
      if: github.ref == 'refs/heads/develop'
      run: |
        git checkout ${{ steps.configuration.outputs.alpha_version_api }}
        echo "Current branch is ${{ github.ref }}"
    - name: "Import Vault Secrets for Alpha Test Runner Service Account"
      uses: hashicorp/vault-action@v2.1.0
      with:
        url: ${{ secrets.VAULT_ADDR }}
        method: approle
        roleId: ${{ secrets.ALPHA_ROLE_ID }}
        secretId: ${{ secrets.ALPHA_SECRET_ID }}
        secrets: |
          secret/dsde/datarepo/alpha/test-runner-sa key | B64_APPLICATION_CREDENTIALS ;
    - name: "Perform IAM policy cleanup for Alpha"
      run: |
        # write vault tokens
        base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}

        gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}

        ./tools/cleanupPolicies.sh ${GOOGLE_CLOUD_DATA_PROJECT}
    - name: "Import Vault Secrets for Dev Service Account"
      uses: hashicorp/vault-action@v2.1.0
      with:
        url: ${{ secrets.VAULT_ADDR }}
        method: approle
        roleId: ${{ secrets.ROLE_ID }}
        secretId: ${{ secrets.SECRET_ID }}
        secrets: |
          secret/dsde/datarepo/dev/sa-key-b64 sa | B64_APPLICATION_CREDENTIALS ;
    - name: "Add jade-k8-sa credentials to run as Harry Potter test users"
      env:
        # note: hack to overwrite the env var to grab the dev credentials too
        GOOGLE_APPLICATION_CREDENTIALS: /tmp/jade-dev-account.json
      run: |
        # write vault tokens
        base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}
    - name: "Build and run Test Runner"
      run: |
        cd ${GITHUB_WORKSPACE}/${workingDir}
        echo "Building Data Repo client library"
        export TEST_RUNNER_SERVER_SPECIFICATION_FILE="alpha.json"
        export TEST_RUNNER_BILLING_PROFILE_NAME="stest1"
        ENABLE_SUBPROJECT_TASKS=1 ./gradlew :datarepo-client:clean :datarepo-client:assemble
        cd ${GITHUB_WORKSPACE}/${workingDir}/datarepo-clienttests
        export ORG_GRADLE_PROJECT_datarepoclientjar=$(find .. -type f -name "datarepo-client*.jar")
        echo "ORG_GRADLE_PROJECT_datarepoclientjar = ${ORG_GRADLE_PROJECT_datarepoclientjar}"

          
        echo "[INFO] Running spotless and spotbugs"
        ./gradlew spotlessCheck
        ./gradlew spotbugsMain

        outputDir="/tmp/TestRunnerResults"
        echo "Output directory set to: $outputDir"
  
        echo "[INFO] Running tests"
        ./gradlew runTest --args="suites/AlphaSmokeTests.json $outputDir" ||
          (echo "Running tests FAILED" &&
          ./gradlew uploadResults --args="BroadJadeDev.json $outputDir" &&
          return 1)
        echo "[INFO] Running test suite SUCCEEDED"

        echo "[INFO] Collecting measurements"
        ./gradlew collectMeasurements --args="AlphaSmokeTests.json $outputDir" ||
          (echo "Collecting measurements FAILED" &&
          ./gradlew uploadResults --args="BroadJadeDev.json $outputDir" &&
          return 1)
        echo "[INFO] Collecting measurements SUCCEEDED"

        echo "[INFO] Uploading results"
        ./gradlew uploadResults --args="BroadJadeDev.json $outputDir"
        echo "[INFO] Uploading results SUCCEEDED"
        cd ${GITHUB_WORKSPACE}/${workingDir}
    - name: "[Cherry-pick to public GCR] Import Vault Secrets for GCR Service Account"
      uses: hashicorp/vault-action@v2.1.0
      with:
        url: ${{ secrets.VAULT_ADDR }}
        method: approle
        roleId: ${{ secrets.ROLE_ID }}
        secretId: ${{ secrets.SECRET_ID }}
        secrets: |
          secret/dsde/datarepo/dev/gcr-sa-b64 key | B64_APPLICATION_CREDENTIALS ;
    - name: "[Cherry-pick to public GCR] Authenticate with GCR SA Credentials"
      env:
        GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcr-sa.json
      run: |
        # write vault tokens
        base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}

        gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}
    - name: "[API][Cherry-pick to public GCR] Perform cherry-pick"
      run: |
        DEV_IMAGE="${GCR_DEV_URL}/${API_REPO}:${{ steps.configuration.outputs.alpha_version_api }}"
        PUBLIC_IMAGE="${GCR_PUBLIC_URL}/${API_REPO}:${{ steps.configuration.outputs.alpha_version_api }}"
        echo "Cherry picking ${{ steps.configuration.outputs.alpha_version_api }} from ${DEV_IMAGE} to ${PUBLIC_IMAGE}"
        gcloud container images add-tag --quiet "${DEV_IMAGE}" "${PUBLIC_IMAGE}"
    - name: "[UI][Cherry-pick to public GCR] Checkout datarepo-helm repo"
      uses: actions/checkout@v2
      with:
        repository: 'broadinstitute/datarepo-helm'
        token: ${{ secrets.HELM_REPO_TOKEN }}
        path: datarepo-helm
    - name: "[UI][Cherry-pick to public GCR] Read UI version from datarepo-helm" 
      id: datarepohelm
      run: |
        alpha_version_ui=$(docker run --rm -v ${PWD}:/workdir mikefarah/yq:3 yq r datarepo-helm/charts/datarepo-ui/values.yaml image.tag)
        echo "::set-output name=alpha_version_ui::$alpha_version_ui"
        echo "Alpha UI Version: $alpha_version_ui"
    - name: "[UI][Cherry-pick to public GCR] Perform cherry-pick"
      run: |
        DEV_IMAGE="${GCR_DEV_URL}/${UI_REPO}:${{ steps.datarepohelm.outputs.alpha_version_ui }}"
        PUBLIC_IMAGE="${GCR_PUBLIC_URL}/${UI_REPO}:${{ steps.datarepohelm.outputs.alpha_version_ui }}"
        echo "Cherry picking ${{ steps.datarepohelm.outputs.alpha_version_ui }} from ${DEV_IMAGE} to ${PUBLIC_IMAGE}"
        gcloud container images add-tag --quiet "${DEV_IMAGE}" "${PUBLIC_IMAGE}"
    - name: "Notify Slack"
      if: always()
      uses: broadinstitute/action-slack@v3.8.0
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      with:
        status: ${{ job.status }}
        fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
        channel: "#jade-alerts"
        username: "Data Repo tests"
        text: "Alpha Smoke tests"<commit_msg>add qa slack notification (#818)<commit_after>name: DataRepo Alpha Tests and Prod GCR Promotion
env:
  K8_CLUSTER: jade-master-us-central1
  GOOGLE_APPLICATION_CREDENTIALS: /tmp/alpha-test-runner.json
  GOOGLE_CLOUD_PROJECT: terra-datarepo-alpha
  GOOGLE_CLOUD_DATA_PROJECT: terra-datarepo-alpha-data
  TDR_LOG_APPENDER: Console-Standard
  GCR_DEV_URL: gcr.io/broad-jade-dev
  GCR_PUBLIC_URL: gcr.io/datarepo-public-gcr
  API_REPO: jade-data-repo
  UI_REPO: jade-data-repo-ui
on:
  # Jenkins will kick this job off manually after Alpha is deployed
  workflow_dispatch: {}
jobs:
  alpha-tests-and-gcr-promotion:
    runs-on: ubuntu-latest
    steps:
    - name: "Checkout jade-data-repo and latest tags"
      uses: actions/checkout@v2
      with:
        fetch-depth: 0
    - name: "Fetch current alpha version from /configuration endpoint"
      id: configuration
      run: |
        alpha_version_api=$(curl -s -X GET "https://data.alpha.envs-terra.bio/configuration" -H "accept: application/json" | jq -r '.semVer|rtrimstr("-SNAPSHOT")')
        echo "::set-output name=alpha_version_api::$alpha_version_api"
        echo "Alpha Version: $alpha_version_api"
    - name: "Checkout tag for DataBiosphere/jade-data-repo"
      if: github.ref == 'refs/heads/develop'
      run: |
        git checkout ${{ steps.configuration.outputs.alpha_version_api }}
        echo "Current branch is ${{ github.ref }}"
    - name: "Import Vault Secrets for Alpha Test Runner Service Account"
      uses: hashicorp/vault-action@v2.1.0
      with:
        url: ${{ secrets.VAULT_ADDR }}
        method: approle
        roleId: ${{ secrets.ALPHA_ROLE_ID }}
        secretId: ${{ secrets.ALPHA_SECRET_ID }}
        secrets: |
          secret/dsde/datarepo/alpha/test-runner-sa key | B64_APPLICATION_CREDENTIALS ;
    - name: "Perform IAM policy cleanup for Alpha"
      run: |
        # write vault tokens
        base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}

        gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}

        ./tools/cleanupPolicies.sh ${GOOGLE_CLOUD_DATA_PROJECT}
    - name: "Import Vault Secrets for Dev Service Account"
      uses: hashicorp/vault-action@v2.1.0
      with:
        url: ${{ secrets.VAULT_ADDR }}
        method: approle
        roleId: ${{ secrets.ROLE_ID }}
        secretId: ${{ secrets.SECRET_ID }}
        secrets: |
          secret/dsde/datarepo/dev/sa-key-b64 sa | B64_APPLICATION_CREDENTIALS ;
    - name: "Add jade-k8-sa credentials to run as Harry Potter test users"
      env:
        # note: hack to overwrite the env var to grab the dev credentials too
        GOOGLE_APPLICATION_CREDENTIALS: /tmp/jade-dev-account.json
      run: |
        # write vault tokens
        base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}
    - name: "Build and run Test Runner"
      run: |
        cd ${GITHUB_WORKSPACE}/${workingDir}
        echo "Building Data Repo client library"
        export TEST_RUNNER_SERVER_SPECIFICATION_FILE="alpha.json"
        export TEST_RUNNER_BILLING_PROFILE_NAME="stest1"
        ENABLE_SUBPROJECT_TASKS=1 ./gradlew :datarepo-client:clean :datarepo-client:assemble
        cd ${GITHUB_WORKSPACE}/${workingDir}/datarepo-clienttests
        export ORG_GRADLE_PROJECT_datarepoclientjar=$(find .. -type f -name "datarepo-client*.jar")
        echo "ORG_GRADLE_PROJECT_datarepoclientjar = ${ORG_GRADLE_PROJECT_datarepoclientjar}"

          
        echo "[INFO] Running spotless and spotbugs"
        ./gradlew spotlessCheck
        ./gradlew spotbugsMain

        outputDir="/tmp/TestRunnerResults"
        echo "Output directory set to: $outputDir"
  
        echo "[INFO] Running tests"
        ./gradlew runTest --args="suites/AlphaSmokeTests.json $outputDir" ||
          (echo "Running tests FAILED" &&
          ./gradlew uploadResults --args="BroadJadeDev.json $outputDir" &&
          return 1)
        echo "[INFO] Running test suite SUCCEEDED"

        echo "[INFO] Collecting measurements"
        ./gradlew collectMeasurements --args="AlphaSmokeTests.json $outputDir" ||
          (echo "Collecting measurements FAILED" &&
          ./gradlew uploadResults --args="BroadJadeDev.json $outputDir" &&
          return 1)
        echo "[INFO] Collecting measurements SUCCEEDED"

        echo "[INFO] Uploading results"
        ./gradlew uploadResults --args="BroadJadeDev.json $outputDir"
        echo "[INFO] Uploading results SUCCEEDED"
        cd ${GITHUB_WORKSPACE}/${workingDir}
    - name: "[Cherry-pick to public GCR] Import Vault Secrets for GCR Service Account"
      uses: hashicorp/vault-action@v2.1.0
      with:
        url: ${{ secrets.VAULT_ADDR }}
        method: approle
        roleId: ${{ secrets.ROLE_ID }}
        secretId: ${{ secrets.SECRET_ID }}
        secrets: |
          secret/dsde/datarepo/dev/gcr-sa-b64 key | B64_APPLICATION_CREDENTIALS ;
    - name: "[Cherry-pick to public GCR] Authenticate with GCR SA Credentials"
      env:
        GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcr-sa.json
      run: |
        # write vault tokens
        base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS}

        gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}
    - name: "[API][Cherry-pick to public GCR] Perform cherry-pick"
      run: |
        DEV_IMAGE="${GCR_DEV_URL}/${API_REPO}:${{ steps.configuration.outputs.alpha_version_api }}"
        PUBLIC_IMAGE="${GCR_PUBLIC_URL}/${API_REPO}:${{ steps.configuration.outputs.alpha_version_api }}"
        echo "Cherry picking ${{ steps.configuration.outputs.alpha_version_api }} from ${DEV_IMAGE} to ${PUBLIC_IMAGE}"
        gcloud container images add-tag --quiet "${DEV_IMAGE}" "${PUBLIC_IMAGE}"
    - name: "[UI][Cherry-pick to public GCR] Checkout datarepo-helm repo"
      uses: actions/checkout@v2
      with:
        repository: 'broadinstitute/datarepo-helm'
        token: ${{ secrets.HELM_REPO_TOKEN }}
        path: datarepo-helm
    - name: "[UI][Cherry-pick to public GCR] Read UI version from datarepo-helm" 
      id: datarepohelm
      run: |
        alpha_version_ui=$(docker run --rm -v ${PWD}:/workdir mikefarah/yq:3 yq r datarepo-helm/charts/datarepo-ui/values.yaml image.tag)
        echo "::set-output name=alpha_version_ui::$alpha_version_ui"
        echo "Alpha UI Version: $alpha_version_ui"
    - name: "[UI][Cherry-pick to public GCR] Perform cherry-pick"
      run: |
        DEV_IMAGE="${GCR_DEV_URL}/${UI_REPO}:${{ steps.datarepohelm.outputs.alpha_version_ui }}"
        PUBLIC_IMAGE="${GCR_PUBLIC_URL}/${UI_REPO}:${{ steps.datarepohelm.outputs.alpha_version_ui }}"
        echo "Cherry picking ${{ steps.datarepohelm.outputs.alpha_version_ui }} from ${DEV_IMAGE} to ${PUBLIC_IMAGE}"
        gcloud container images add-tag --quiet "${DEV_IMAGE}" "${PUBLIC_IMAGE}"
    - name: "Notify Slack"
      if: always()
      uses: broadinstitute/action-slack@v3.8.0
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      with:
        status: ${{ job.status }}
        fields: job,repo,message,author,took
        channel: "#jade-alerts"
        username: "Data Repo tests"
        text: "Alpha Tests and GCR Promotion"
    - name: "Notify QA Slack"
      if: always()
      uses: broadinstitute/action-slack@v3.8.0
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      with:
        status: ${{ job.status }}
        channel: "#dsde-qa"
        username: "Data Repo tests"
        text: "Alpha Tests and GCR Promotion"
        fields: job,repo,message,author,took
{'fraction_non_alphanumeric': 0.1160844733696374, 'fraction_numerical': 0.0063753486518794, 'mean_word_length': 2.9411148913896885, 'pattern_counts': {'":': 4, '<': 21, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7654619', 'n_tokens_mistral': 5145, 'n_tokens_neox': 4915, 'n_words': 1039}