<commit_before>name: selfdrive
on:
  push:
    branches-ignore:
      - 'testing-closet*'
  pull_request:

env:
  BASE_IMAGE: openpilot-base
  DOCKER_REGISTRY: ghcr.io/commaai

  DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
  BUILD: |
      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
  UNIT_TEST: coverage run --append -m unittest discover

jobs:
  # TODO: once actions/cache supports read only mode, use the cache for all jobs
  build_release:
    name: build release
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    env:
      STRIPPED_DIR: tmppilot
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: release/check-submodules.sh
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Strip non-release files
      run: |
        mkdir $STRIPPED_DIR
        cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
        cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
        cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
        cp Dockerfile.openpilot_base $STRIPPED_DIR

        # need this to build on x86
        cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot and run checks
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
                          python -m unittest discover selfdrive/car"
    - name: Cleanup scons cache
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "scons -j$(nproc) && \
                          rm -rf /tmp/scons_cache/* && \
                          scons -j$(nproc) --cache-populate"

  #build_mac:
  #  name: build macos
  #  runs-on: macos-latest
  #  timeout-minutes: 60
  #  steps:
  #  - uses: actions/checkout@v2
  #    with:
  #      submodules: true
  #  - name: Determine pre-existing Homebrew packages
  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
  #    run: |
  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
  #      echo 'EOF' >> $GITHUB_ENV
  #  - name: Cache dependencies
  #    id: dependency-cache
  #    uses: actions/cache@v2
  #    with:<commit_msg>CI: add job for complete build (#24283)<commit_after>name: selfdrive
on:
  push:
    branches-ignore:
      - 'testing-closet*'
  pull_request:

env:
  BASE_IMAGE: openpilot-base
  DOCKER_REGISTRY: ghcr.io/commaai

  DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
  BUILD: |
      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
  UNIT_TEST: coverage run --append -m unittest discover

jobs:
  # TODO: once actions/cache supports read only mode, use the cache for all jobs
  build_release:
    name: build release
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    env:
      STRIPPED_DIR: tmppilot
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: release/check-submodules.sh
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Strip non-release files
      run: |
        mkdir $STRIPPED_DIR
        cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
        cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
        cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
        cp Dockerfile.openpilot_base $STRIPPED_DIR

        # need this to build on x86
        cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot and run checks
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
                          python -m unittest discover selfdrive/car"
    - name: Cleanup scons cache
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "scons -j$(nproc) && \
                          rm -rf /tmp/scons_cache/* && \
                          scons -j$(nproc) --cache-populate"

  build_all:
    name: build all
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot with all flags
      run: ${{ env.RUN }} "scons -j$(nproc) --extras --test"

  #build_mac:
  #  name: build macos
  #  runs-on: macos-latest
  #  timeout-minutes: 60
  #  steps:
  #  - uses: actions/checkout@v2
  #    with:
  #      submodules: true
  #  - name: Determine pre-existing Homebrew packages
  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
  #    run: |
  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
  #      echo 'EOF' >> $GITHUB_ENV
  #  - name: Cache dependencies
  #    id: dependency-cache
  #    uses: actions/cache@v2
  #    with:
{'fraction_non_alphanumeric': 0.11600547195622435, 'fraction_numerical': 0.014500683994528044, 'mean_word_length': 2.445334590009425, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '656342', 'n_tokens_mistral': 2760, 'n_tokens_neox': 2574, 'n_words': 600}