<commit_before>name: ci-post-land

on:
  create:
    branches: [main, release-*]
  push:
    branches: [main, release-*] # gha-test-*

defaults:
  run:
    shell: bash

env:
  max_threads: 16
  pre_command: cd /opt/git/diem/

jobs:
  prepare:
    runs-on: ubuntu-20.04
    outputs:
      changes-target-branch: ${{ steps.changes.outputs.changes-target-branch }}
      changes-base-git-rev: ${{ steps.changes.outputs.changes-base-git-rev }}
      changes-base-githash: ${{ steps.changes.outputs.changes-base-githash }}
      changes-pull-request-number: ${{ steps.changes.outputs.changes-pull-request-number }}
      rust-changes: ${{ steps.rust-changes.outputs.changes-found }}
      base-image-changes: ${{ steps.base-image-changes.outputs.changes-found }}
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - name: Git Hooks and Checks
        run: ./scripts/git-checks.sh
      - id: changes
        name: determine changes
        uses: diem/actions/changes@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          workflow-file: ci-post-land.yml
          github-token: ${{secrets.GITHUB_TOKEN}}
      - id: rust-changes
        name: find rust/cargo changes.
        uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          pattern: '^documentation\|^docker\|^scripts\|^developers.di<EMAIL>'
          invert: "true"
      - id: base-image-changes
        name: find dev-setup.sh/base docker image changes
        uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          pattern: '.github/workflows/ci-post-land.yml\|.github/actions/dockerhub_login/action.yml\|docker/ci/github/Dockerfile\|scripts/dev_setup.sh\|rust-toolchain'

  # NOTE: update-sscache-osx removed as cost savings effort. It strongly resembled update-sccache-ubuntu
  # except for a 'runs-on: macos-11', and no container.  Restore when there are more osx contributors.

  update-sccache-ubuntu:
    needs: prepare
    if: ${{ needs.prepare.outputs.rust-changes == 'true' }}
    environment:
      name: Sccache
    runs-on: ubuntu-20.04-xl
    container:
      image: ghcr.io/diem/diem_build_environment:${{ needs.prepare.outputs.changes-target-branch }}
      volumes:
        - "${{github.workspace}}:/opt/git/diem"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - uses: ./.github/actions/build-setup
      - uses: actions/cache@v2.1.6
        with:
          path: "/opt/cargo/git\n/opt/cargo/registry\n/opt/cargo/.package-cache"
          key: crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
          restore-keys: "crates-${{ runner.os }}"
      - name: build all unit test code.
        run: |
          $pre_command && cargo x test --no-run --jobs ${max_threads} --unit
        env:
          TARGET_BRANCH: ${{ needs.prepare.outputs.changes-target-branch }}
          SCCACHE_AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }}
          SCCACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }}
      - uses: ./.github/actions/build-teardown

  # In the future when the script can differentiate between target branches for each auto run then we'll
  # update this report to differentiate branches.
  unit-test-allure-report:
    name: Unit Test Reports
    runs-on: ubuntu-20.04
    if: ${{ github.ref == 'refs/heads/main' }}
    environment:
      name: Sccache
    container:
      image: ghcr.io/diem/diem_build_environment:main
      volumes:
        - "${{github.workspace}}:/opt/git/diem"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - uses: ./.github/actions/build-setup
        # build setup is called to ensure the latest docker image tooling is in place and the ~/.profile is called.
      - name: Publish unit test results
        uses: diem/actions/process-test-results@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          #used to read artifacts, annoying to need one.
          token: ${{ secrets.GITHUB_TOKEN }}
          artifacts: "unit-test-results codegen-unit-test-results"
          workflow_file: ci-test.yml
          branch: auto
          history: 20
          recreate: false
          allure_configuration: ./allure-configuration
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: ${{ secrets.ENV_DIEM_S3_AWS_REGION }}

  build_ci_base_docker_image:
    needs: prepare
    runs-on: ubuntu-20.04-xl
    if: ${{ needs.prepare.outputs.base-image-changes == 'true' || github.event_name == 'create' }}
    continue-on-error: false
    env:
      DOCKERHUB_ORG: diem
    environment:
      name: Docker
      url: https://hub.docker.com/u/diem
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - name: Git Hooks and Checks
        run: ./scripts/git-checks.sh
      - name: build image
        run: docker build -f docker/ci/github/Dockerfile -t ${{ env.DOCKERHUB_ORG }}/build_environment:${{ needs.prepare.outputs.changes-target-branch }} .
      - name: Sign in to dockerhub, install image signing cert.
        uses: ./.github/actions/dockerhub_login
        with:
          username: ${{ secrets.ENV_DOCKERHUB_USERNAME }}
          password: ${{ secrets.ENV_DOCKERHUB_PASSWORD }}
          key_material: ${{ secrets.ENV_DOCKERHUB_KEY_MATERIAL }}
          key_name: ${{ secrets.ENV_DOCKERHUB_KEY_NAME }}
          key_password: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }}
      - name: Push to dockerhub.
        run: |
          if [[ "$DOCKERHUB_LOGGED_IN" == true ]]; then
            disable_content_trust=true
            if [[ "$DOCKERHUB_CAN_SIGN" == true ]]; then
              disable_content_trust=false
            fi
            docker push --disable-content-trust=${disable_content_trust} ${{ env.DOCKERHUB_ORG }}/build_environment:${{ needs.prepare.outputs.changes-target-branch }}
          fi
        env:
          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }}

  build_ci_base_docker_image_github:
    needs: prepare
    runs-on: ubuntu-latest-xl
    if: ${{ needs.prepare.outputs.base-image-changes == 'true' || github.event_name == 'create' }}
    env:
      REGISTRY: ghcr.io
      IMAGE_NAME: ${{ github.repository }}_build_environment
      IMAGE_TAG: ${{ needs.prepare.outputs.changes-target-branch }}
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - name: Log in to the Container registry
        uses: docker/login-action@v1
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ <PASSWORD> }}
      - name: build image
        run: docker build -f docker/ci/github/Dockerfile -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} .
      - name: Push to ghcr
        run: |
          docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}

  run-cluster-test-pre-release-suite:
    needs: prepare
    name: Run the pre-release suite of Cluster Test
    runs-on: self-hosted
    if: ${{  startsWith(needs.prepare.outputs.changes-target-branch, 'release') }}
    # The pre-release suite run time varies 1~1.5 hr.<commit_msg>ci: remove old dockerhub base image job<commit_after>name: ci-post-land

on:
  create:
    branches: [main, release-*]
  push:
    branches: [main, release-*] # gha-test-*

defaults:
  run:
    shell: bash

env:
  max_threads: 16
  pre_command: cd /opt/git/diem/

jobs:
  prepare:
    runs-on: ubuntu-20.04
    outputs:
      changes-target-branch: ${{ steps.changes.outputs.changes-target-branch }}
      changes-base-git-rev: ${{ steps.changes.outputs.changes-base-git-rev }}
      changes-base-githash: ${{ steps.changes.outputs.changes-base-githash }}
      changes-pull-request-number: ${{ steps.changes.outputs.changes-pull-request-number }}
      rust-changes: ${{ steps.rust-changes.outputs.changes-found }}
      base-image-changes: ${{ steps.base-image-changes.outputs.changes-found }}
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - name: Git Hooks and Checks
        run: ./scripts/git-checks.sh
      - id: changes
        name: determine changes
        uses: diem/actions/changes@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          workflow-file: ci-post-land.yml
          github-token: ${{secrets.GITHUB_TOKEN}}
      - id: rust-changes
        name: find rust/cargo changes.
        uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          pattern: '^documentation\|^docker\|^scripts\|^developers.diem.com'
          invert: "true"
      - id: base-image-changes
        name: find dev-setup.sh/base docker image changes
        uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          pattern: '.github/workflows/ci-post-land.yml\|.github/actions/dockerhub_login/action.yml\|docker/ci/github/Dockerfile\|scripts/dev_setup.sh\|rust-toolchain'

  # NOTE: update-sscache-osx removed as cost savings effort. It strongly resembled update-sccache-ubuntu
  # except for a 'runs-on: macos-11', and no container.  Restore when there are more osx contributors.

  update-sccache-ubuntu:
    needs: prepare
    if: ${{ needs.prepare.outputs.rust-changes == 'true' }}
    environment:
      name: Sccache
    runs-on: ubuntu-20.04-xl
    container:
      image: ghcr.io/diem/diem_build_environment:${{ needs.prepare.outputs.changes-target-branch }}
      volumes:
        - "${{github.workspace}}:/opt/git/diem"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - uses: ./.github/actions/build-setup
      - uses: actions/cache@v2.1.6
        with:
          path: "/opt/cargo/git\n/opt/cargo/registry\n/opt/cargo/.package-cache"
          key: crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
          restore-keys: "crates-${{ runner.os }}"
      - name: build all unit test code.
        run: |
          $pre_command && cargo x test --no-run --jobs ${max_threads} --unit
        env:
          TARGET_BRANCH: ${{ needs.prepare.outputs.changes-target-branch }}
          SCCACHE_AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }}
          SCCACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }}
      - uses: ./.github/actions/build-teardown

  # In the future when the script can differentiate between target branches for each auto run then we'll
  # update this report to differentiate branches.
  unit-test-allure-report:
    name: Unit Test Reports
    runs-on: ubuntu-20.04
    if: ${{ github.ref == 'refs/heads/main' }}
    environment:
      name: Sccache
    container:
      image: ghcr.io/diem/diem_build_environment:main
      volumes:
        - "${{github.workspace}}:/opt/git/diem"
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - uses: ./.github/actions/build-setup
        # build setup is called to ensure the latest docker image tooling is in place and the ~/.profile is called.
      - name: Publish unit test results
        uses: diem/actions/process-test-results@faadd16607b77dfa2231a8f366883e01717b3225
        with:
          #used to read artifacts, annoying to need one.
          token: ${{ secrets.GITHUB_TOKEN }}
          artifacts: "unit-test-results codegen-unit-test-results"
          workflow_file: ci-test.yml
          branch: auto
          history: 20
          recreate: false
          allure_configuration: ./allure-configuration
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: ${{ secrets.ENV_DIEM_S3_AWS_REGION }}

  build_ci_base_docker_image:
    needs: prepare
    runs-on: ubuntu-20.04-xl
    if: ${{ needs.prepare.outputs.base-image-changes == 'true' || github.event_name == 'create' }}
    env:
      REGISTRY: ghcr.io
      IMAGE_NAME: ${{ github.repository }}_build_environment
      IMAGE_TAG: ${{ needs.prepare.outputs.changes-target-branch }}
    steps:
      - uses: actions/checkout@v2.3.4
        with:
          # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
          # On `push` this value will be empty and will "do-the-right-thing"
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0 #get all the history!!!
      - name: Log in to the Container registry
        uses: docker/login-action@v1
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ <PASSWORD> }}
      - name: build image
        run: docker build -f docker/ci/github/Dockerfile -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} .
      - name: Push to ghcr
        run: |
          docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}

  run-cluster-test-pre-release-suite:
    needs: prepare
    name: Run the pre-release suite of Cluster Test
    runs-on: self-hosted
    if: ${{  startsWith(needs.prepare.outputs.changes-target-branch, 'release') }}
    # The pre-release suite run time varies 1~1.5 hr.
{'fraction_non_alphanumeric': 0.11933814604229222, 'fraction_numerical': 0.020443365489043634, 'mean_word_length': 2.4809873248832557, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '107501', 'n_tokens_mistral': 5300, 'n_tokens_neox': 4927, 'n_words': 1178}