<commit_before>name: Run tests with twister

on:
  push:
    branches:
      - main
      - v*-branch
  pull_request_target:
    branches:
      - main
      - v*-branch
  schedule:
    # Run at 00:00 on Wednesday and Saturday
    - cron: '0 0 * * 3,6'

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  twister-build-prep:
    if: github.repository_owner == 'zephyrproject-rtos'
    runs-on: zephyr-runner-linux-x64-4xlarge
    container:
      image: ghcr.io/zephyrproject-rtos/ci:v0.24.5
      options: '--entrypoint /bin/bash'
      volumes:
        - /repo-cache/zephyrproject:/github/cache/zephyrproject
    outputs:
      subset: ${{ steps.output-services.outputs.subset }}
      size: ${{ steps.output-services.outputs.size }}
      fullrun: ${{ steps.output-services.outputs.fullrun }}
    env:
      MATRIX_SIZE: 10
      PUSH_MATRIX_SIZE: 15
      DAILY_MATRIX_SIZE: 80
      ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.15.1
      TESTS_PER_BUILDER: 700
      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
      BASE_REF: ${{ github.base_ref }}
    steps:
      - name: Apply container owner mismatch workaround
        run: |
          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
          #        match the container user UID because of the way GitHub
          #        Actions runner is implemented. Remove this workaround when
          #        GitHub comes up with a fundamental fix for this problem.
          git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Clone cached Zephyr repository
        if: github.event_name == 'pull_request_target'
        continue-on-error: true
        run: |
          git clone /github/cache/zephyrproject/zephyr .
          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}

      - name: Checkout
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
          persist-credentials: false

      - name: Environment Setup
        if: github.event_name == 'pull_request_target'
        run: |
          git config --global user.email "<EMAIL>"
          git config --global user.name "Zephyr Bot"
          rm -fr ".git/rebase-apply"
          git rebase origin/${BASE_REF}
          git log  --pretty=oneline | head -n 10
          west init -l . || true
          west config manifest.group-filter -- +ci
          west config --global update.narrow true
          west update --path-cache /github/cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /github/cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /github/cache/zephyrproject)
          west forall -c 'git reset --hard HEAD'

      - name: Generate Test Plan with Twister
        if: github.event_name == 'pull_request_target'
        id: test-plan
        run: |
          export ZEPHYR_BASE=${PWD}
          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
          if [ -s .testplan ]; then
            cat .testplan >> $GITHUB_ENV
          else
            echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
          fi
          rm -f testplan.json .testplan

      - name: Determine matrix size
        id: output-services
        run: |
          if [ "${{github.event_name}}" = "pull_request_target" ]; then
            if [ -n "${TWISTER_NODES}" ]; then
              subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
            else
              subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
            fi
            size=${TWISTER_NODES}
          elif [ "${{github.event_name}}" = "push" ]; then
            subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
            size=${MATRIX_SIZE}
          elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
            subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
            size=${DAILY_MATRIX_SIZE}
          else
            size=0
          fi
          echo "subset=${subset}" >> $GITHUB_OUTPUT
          echo "size=${size}" >> $GITHUB_OUTPUT
          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT

  twister-build:
    runs-on: zephyr-runner-linux-x64-4xlarge
    needs: twister-build-prep
    if: needs.twister-build-prep.outputs.size != 0
    container:
      image: ghcr.io/zephyrproject-rtos/ci:v0.24.5
      options: '--entrypoint /bin/bash'
      volumes:
        - /repo-cache/zephyrproject:/github/cache/zephyrproject
    strategy:
      fail-fast: false
      matrix:
        subset: ${{fromJSON(needs.twister-build-prep.outputs.subset)}}
    env:
      ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.15.1
      TWISTER_COMMON: ' --force-color --inline-logs -v -N -M --retry-failed 3 '
      DAILY_OPTIONS: ' -M --build-only --all'
      PR_OPTIONS: ' --clobber-output --integration'
      PUSH_OPTIONS: ' --clobber-output -M'
      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
      BASE_REF: ${{ github.base_ref }}
    steps:
      - name: Apply container owner mismatch workaround
        run: |
          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
          #        match the container user UID because of the way GitHub
          #        Actions runner is implemented. Remove this workaround when
          #        GitHub comes up with a fundamental fix for this problem.
          git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Clone cached Zephyr repository
        continue-on-error: true
        run: |
          git clone /github/cache/zephyrproject/zephyr .
          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}

      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
          persist-credentials: false

      - name: Environment Setup
        run: |
          if [ "${{github.event_name}}" = "pull_request_target" ]; then
            git config --global user.email "<EMAIL>"<commit_msg>ci: twister: capture footprint data on push/schedule<commit_after>name: Run tests with twister

on:
  push:
    branches:
      - main
      - v*-branch
  pull_request_target:
    branches:
      - main
      - v*-branch
  schedule:
    # Run at 00:00 on Wednesday and Saturday
    - cron: '0 0 * * 3,6'

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:
  twister-build-prep:
    if: github.repository_owner == 'zephyrproject-rtos'
    runs-on: zephyr-runner-linux-x64-4xlarge
    container:
      image: ghcr.io/zephyrproject-rtos/ci:v0.24.5
      options: '--entrypoint /bin/bash'
      volumes:
        - /repo-cache/zephyrproject:/github/cache/zephyrproject
    outputs:
      subset: ${{ steps.output-services.outputs.subset }}
      size: ${{ steps.output-services.outputs.size }}
      fullrun: ${{ steps.output-services.outputs.fullrun }}
    env:
      MATRIX_SIZE: 10
      PUSH_MATRIX_SIZE: 15
      DAILY_MATRIX_SIZE: 80
      ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.15.1
      TESTS_PER_BUILDER: 700
      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
      BASE_REF: ${{ github.base_ref }}
    steps:
      - name: Apply container owner mismatch workaround
        run: |
          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
          #        match the container user UID because of the way GitHub
          #        Actions runner is implemented. Remove this workaround when
          #        GitHub comes up with a fundamental fix for this problem.
          git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Clone cached Zephyr repository
        if: github.event_name == 'pull_request_target'
        continue-on-error: true
        run: |
          git clone /github/cache/zephyrproject/zephyr .
          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}

      - name: Checkout
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
          persist-credentials: false

      - name: Environment Setup
        if: github.event_name == 'pull_request_target'
        run: |
          git config --global user.email "<EMAIL>"
          git config --global user.name "<NAME>"
          rm -fr ".git/rebase-apply"
          git rebase origin/${BASE_REF}
          git log  --pretty=oneline | head -n 10
          west init -l . || true
          west config manifest.group-filter -- +ci
          west config --global update.narrow true
          west update --path-cache /github/cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /github/cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /github/cache/zephyrproject)
          west forall -c 'git reset --hard HEAD'

      - name: Generate Test Plan with Twister
        if: github.event_name == 'pull_request_target'
        id: test-plan
        run: |
          export ZEPHYR_BASE=${PWD}
          export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
          python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
          if [ -s .testplan ]; then
            cat .testplan >> $GITHUB_ENV
          else
            echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
          fi
          rm -f testplan.json .testplan

      - name: Determine matrix size
        id: output-services
        run: |
          if [ "${{github.event_name}}" = "pull_request_target" ]; then
            if [ -n "${TWISTER_NODES}" ]; then
              subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
            else
              subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
            fi
            size=${TWISTER_NODES}
          elif [ "${{github.event_name}}" = "push" ]; then
            subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
            size=${MATRIX_SIZE}
          elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
            subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
            size=${DAILY_MATRIX_SIZE}
          else
            size=0
          fi
          echo "subset=${subset}" >> $GITHUB_OUTPUT
          echo "size=${size}" >> $GITHUB_OUTPUT
          echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT

  twister-build:
    runs-on: zephyr-runner-linux-x64-4xlarge
    needs: twister-build-prep
    if: needs.twister-build-prep.outputs.size != 0
    container:
      image: ghcr.io/zephyrproject-rtos/ci:v0.24.5
      options: '--entrypoint /bin/bash'
      volumes:
        - /repo-cache/zephyrproject:/github/cache/zephyrproject
    strategy:
      fail-fast: false
      matrix:
        subset: ${{fromJSON(needs.twister-build-prep.outputs.subset)}}
    env:
      ZEPHYR_SDK_INSTALL_DIR: /opt/toolchains/zephyr-sdk-0.15.1
      TWISTER_COMMON: ' --force-color --inline-logs -v -N -M --retry-failed 3 '
      DAILY_OPTIONS: ' -M --build-only --all --show-footprint'
      PR_OPTIONS: ' --clobber-output --integration'
      PUSH_OPTIONS: ' --clobber-output -M --show-footprint'
      COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
      BASE_REF: ${{ github.base_ref }}
    steps:
      - name: Apply container owner mismatch workaround
        run: |
          # FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
          #        match the container user UID because of the way GitHub
          #        Actions runner is implemented. Remove this workaround when
          #        GitHub comes up with a fundamental fix for this problem.
          git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Clone cached Zephyr repository
        continue-on-error: true
        run: |
          git clone /github/cache/zephyrproject/zephyr .
          git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}

      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
          persist-credentials: false

      - name: Environment Setup
        run: |
          if [ "${{github.event_name}}" = "pull_request_target" ]; then
            git config --global user.email "<EMAIL>"
{'fraction_non_alphanumeric': 0.12314965511896458, 'fraction_numerical': 0.009145160040300705, 'mean_word_length': 2.1131483715319663, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4986053', 'n_tokens_mistral': 4248, 'n_tokens_neox': 4036, 'n_words': 942}