MAINT: Upgrade the remaining GitHub Actions. <!-- Thank you for contributing to Pyodide! All improvements are welcome,
     so don't be afraid to make a PR. -->

### Description

<!-- Please explain what your PR is about:
     - reasoning for the change
     - some details of updated code
     - any noteworthy choices to be aware of
	Please refer to any related issues by #<issue_id> -->

- https://github.com/actions/checkout/releases
- https://github.com/docker/build-push-action/releases
- https://github.com/docker/login-action/releases
- https://github.com/docker/metadata-action/releases
- https://github.com/pypa/gh-action-pypi-publish/releases
- https://github.com/test-summary/action/releases

Also, add a final check to the CPython upgrade instructions.

### Checklists

<!-- Note:
     If you think some of these steps are not necessary for your PR,
     remove those checkboxes, or mark them as checked. If you keep unchecked checkboxes,
     we will assume that your PR is not ready to be merged  -->

- [ ] Add a [CHANGELOG](https://github.com/pyodide/pyodide/blob/main/docs/project/changelog.md) entry
- [x] Add / update tests
- [ ] Add new / update outdated documentation

version: 2.1

defaults: &defaults
  working_directory: ~/repo
  docker:
    # Note: when updating the docker image version,
    #       make sure there are no extra old versions lying around.
    #       (e.g. `rg -F --hidden <old_tag>`)
    - image: pyodide/pyodide-env:20230301-chrome109-firefox109-py311
  environment:
    - EMSDK_NUM_CORES: 3
      EMCC_CORES: 3
      PYODIDE_JOBS: 3
      # Make sure the ccache dir is consistent between core and package builds
      # (it's not the case otherwise)
      CCACHE_DIR: /root/.ccache/

orbs:
  macos: circleci/macos@2.2.0

jobs:
  build-core:
    <<: *defaults
    steps:
      - checkout

      - restore_cache:
          keys:
            - -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
            - -core-v20220915-{{ checksum "cpython/Makefile" }}
            - -core-v20220915

      - run:
          name: calculate build hash for ccache
          command: |
            pip install pathspec
            ./tools/calculate_build_cache_key.py > build_hash.txt

      - run:
          name: build emsdk
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C emsdk
            ccache -s

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

      - run:
          name: build cpython
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C cpython
            ccache -s

      - run:
          name: build pyodide core
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            PYODIDE_PACKAGES="tag:core" make
            ccache -s

      - run:
          name: check-size
          command: |
            du dist/ -abh --max-depth 1  | sort -k 2

            pip install brotli
            ./tools/check_compressed_size.py dist/pyodide.asm.* dist/python_stdlib*

      - save_cache:
          paths:
            - /root/.ccache
          key: -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}

      - run:
          name: Clean up workspace
          command: |
            rm -rf cpython/{build,downloads}
            rm -rf emsdk/emsdk/binaryen

      - persist_to_workspace:
          root: .
          paths:
            - .

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide.tar.gz dist
            tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm,_py.tar} dist/{package,repodata}.json dist/python_stdlib.zip

      - store_artifacts:
          path: /root/repo/dist/

      - store_artifacts:
          path: /root/repo/pyodide.tar.gz

      - store_artifacts:
          path: /root/repo/pyodide-core.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

  build-packages:
    parameters:
      packages:
        description: The packages to be built.
        type: string
    <<: *defaults
    resource_class: large
    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>

      - run:
          name: build packages
          no_output_timeout: 60m
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='<< parameters.packages >>' make dist/repodata.json
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: du dist/ -abh --max-depth 1  | sort -k 2

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide.tar.gz dist
            tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm,_py.tar} dist/{package,repodata}.json dist/python_stdlib.zip
            tar cjf build-logs.tar.gz  packages/build-logs

      - run:
          name: Clean up package source files
          command: |
            cd packages && find **/build ! -name '.packaged' -type f -exec rm -f {} +

      - store_artifacts:
          path: /root/repo/pyodide.tar.gz

      - store_artifacts:
          path: /root/repo/pyodide-core.tar.gz

      - store_artifacts:
          path: /root/repo/build-logs.tar.gz

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>

  build-pyodide-debug:
    <<: *defaults
    resource_class: large
    steps:
      - checkout

      - attach_workspace:
          at: .

      - run:
          name: build pyodide debug
          command: |
            cp -r dist dist-release
            rm dist/pyodide.asm.js
            source pyodide_env.sh
            ccache -z
            PYODIDE_DEBUG=1 make dist/pyodide.asm.js
            ccache -s
            cd dist
            npx prettier -w pyodide.asm.js
            npx prettier -w pyodide.js
            cd ..
            mv dist dist-debug
            mv dist-release dist

      - persist_to_workspace:
          root: .
          paths:
            - .

  create-xbuild-env:
    <<: *defaults
    steps:
      - checkout

      - attach_workspace:
          at: .

      - run:
          name: create xbuild environment
          command: |
            pip install -e ./pyodide-build
            python -m pyodide_build create_xbuildenv

      - run:
          name: Zip xbuild environment
          command: |
            tar cjf xbuildenv.tar.gz  xbuildenv/

      - store_artifacts:
          path: /root/repo/xbuildenv.tar.gz

      - persist_to_workspace:
          root: .
          paths:
            - ./xbuildenv

  test-main:
    parameters:
      test-params:
        description: The tests to run.
        type: string
      cache-dir:
        description: pytest-cache-dir.
        type: string
        default: ""
    <<: *defaults
    resource_class: medium+
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            make npm-link
            mkdir test-results
            pip install -r requirements.txt
            pip install -e ./pyodide-build
            npm install -g node-fetch@2
            if [ -z "<< parameters.cache-dir >>" ]; then
              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
            else
              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
            fi
            echo "pytest cache dir: $CACHE_DIR"
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 50 \
              --benchmark-json=benchmark-time.json \
              --benchmark-columns=mean,min,max,stddev \
              << parameters.test-params >> \
              -o cache_dir=$CACHE_DIR
      - store_test_results:
          path: test-results

  test-main-macos:
    parameters:
      test-params:
        description: The tests to run.
        type: string
      cache-dir:
        description: pytest-cache-dir.
        type: string
        default: ""
    resource_class: medium
    macos:
      xcode: 13.3.1

    working_directory: ~/repo
    steps:
      - attach_workspace:
          at: .
      # The standard way of enabling safaridriver no longer works for Safari 14+
      # https://blog.bytesguy.com/enabling-remote-automation-in-safari-14
      - macos/add-safari-permissions
      - run:
          name: install miniforge
          command: |
            curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
            bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh
            ~/miniforge3/bin/conda create -n pyodide python=3.11 -y
      - run:
          name: install dependencies
          command: |
            export PATH="$HOME/miniforge3/bin:$PATH"
            conda create -n pyodide python=3.11 -y
            source activate pyodide
            python -m pip install -r requirements.txt
            pip install -e ./pyodide-build
      - run:
          name: test safari
          command: |
            export PATH="$HOME/miniforge3/bin:$PATH"
            source activate pyodide
            mkdir test-results
            if [ -z "<< parameters.cache-dir >>" ]; then
              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
            else
              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
            fi
            echo "pytest cache dir: $CACHE_DIR"
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 50 \
              << parameters.test-params >> \
              -o cache_dir=$CACHE_DIR
      - store_test_results:
          path: test-results

  benchmark-stack-size:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .
      - run:
          name: stack-size
          command: |
            make npm-link
            pip install -r requirements.txt
            npm install -g node-fetch@2
            pytest -s benchmark/stack_usage.py --rt node chrome firefox | sed -n 's/## //pg'

  test-js:
    <<: *defaults
    resource_class: small
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            cd src/test-js
            npm ci
            npm link ../../dist
            npm run test-types
            npm test
      # - run:
      #     name: check if Pyodide works with Webpack
      #     command: |
      #       git clone https://github.com/pyodide/pyodide-webpack-example.git
      #       export DEV_PYODIDE_PATH=`realpath dist`
      #       cd pyodide-webpack-example
      #       git checkout 164054a9c6fbd2176f386b6552ed8d079c6bcc04
      #       ./build.sh

      - run:
          name: test npm deploy (dry run)
          command: |
            DRY_RUN=1 ./tools/deploy_to_npm.sh

  test-cmdline-runner:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .

      - run:
          name: test
          command: |
            set -x
            export PYODIDE_ROOT=$(pwd)
            echo $PYODIDE_ROOT
            cd ~
            mkdir test
            cd test
            git clone https://github.com/python-attrs/attrs --depth 1 --branch 22.1.0
            git clone https://github.com/zopefoundation/zope.interface.git --depth 1 --branch 5.4.0

            python -m venv .venv-host
            source .venv-host/bin/activate
            pip install $PYODIDE_ROOT/pyodide-build
            pyodide venv .venv-pyodide
            source .venv-pyodide/bin/activate

            cd zope.interface
            pyodide build
            pip install dist/*.whl
            cd ../attrs
            pip install .[tests]
            python -m pytest -k 'not mypy'

  benchmark:
    <<: *defaults
    resource_class: medium+
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: install requirements
          command: |
            pip3 install numpy matplotlib pandas
            pip install -r requirements.txt
            npm install -g node-fetch@2
      - run:
          name: benchmark
          command: |
            python benchmark/benchmark.py all --output dist/benchmarks.json

      - store_artifacts:
          path: /root/repo/dist/benchmarks.json

  deploy-release:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-dev!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make npm
            python3 -m pip install awscli

      - run:
          name: Deploy Github Releases
          command: |
            mkdir -p /tmp/ghr/dist
            cp -r dist /tmp/ghr/pyodide
            cp -r xbuildenv /tmp/ghr/xbuildenv
            cd /tmp/ghr
            tar cjf dist/pyodide-${CIRCLE_TAG}.tar.bz2  pyodide/
            tar cjf dist/pyodide-core-${CIRCLE_TAG}.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm,_py.tar} pyodide/{package,repodata}.json pyodide/python_stdlib.zip
            tar cjf dist/xbuildenv-${CIRCLE_TAG}.tar.bz2  xbuildenv/
            ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" \
              -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" \
              -delete "${CIRCLE_TAG}" \
              dist

      - run:
          name: Deploy to npm
          command: |
            ./tools/deploy_to_npm.sh

      - run:
          name: Set PYODIDE_BASE_URL
          command: |
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/debug/" make dist/console.html
            cp dist/console.html dist-debug/console.html
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make dist/console.html
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip'    # 1 year cache
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year
      - run:
          name: Deploy debug version to pyodide-cdn2.iodide.io
          command: |
            find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*.data' --cache-control 'max-age=30758400, public' --content-encoding 'gzip'  # 1 year cache
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year cache
      - run:
          name: update 301 redirect for the /latest/* route.
          command: |
            aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json

  deploy-dev:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-release!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Set PYODIDE_BASE_URL
          command: |
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/debug/" make dist/console.html
            cp dist/console.html dist-debug/console.html
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make dist/console.html
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache
      - run:
          name: Deploy debug version to pyodide-cdn2.iodide.io
          command: |
            find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/debug/"
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache

workflows:
  version: 2
  build-and-deploy:
    jobs:
      - build-core:
          filters:
            tags:
              only: /.*/

      - build-packages:
          name: build-packages-no-numpy-dependents
          packages: "*,no-numpy-dependents"
          requires:
            - build-core
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages
                  - ./dist

      - build-packages:
          name: build-packages-opencv-python
          packages: opencv-python
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages/opencv-python/build
                  - ./packages/opencv-python/dist
                  - ./packages/build-logs/opencv*
                  - ./dist/opencv*

      - build-packages:
          name: build-packages-numpy-dependents
          packages: "*,!opencv-python"
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages
                  - ./dist

      - build-packages:
          name: build-packages
          packages: "*"
          requires:
            - build-packages-numpy-dependents
            - build-packages-opencv-python
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages/.artifacts
                  - ./dist

      - test-main:
          name: test-core-chrome
          test-params: --runtime=chrome-no-host -k "not webworker" src packages/micropip  packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox
          test-params: --runtime=firefox-no-host -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-node
          test-params: --runtime=node-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-build/pyodide_build/tests
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main-macos:
          name: test-core-safari
          test-params: --runtime=safari-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome-webworker
          test-params: --runtime=chrome-no-host src/tests/test_webworker.py
          requires:
            - test-core-chrome
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox-webworker
          test-params: --runtime=firefox-no-host src/tests/test_webworker.py
          requires:
            - test-core-firefox
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-chrome-no-numpy-dependents
          test-params: --runtime=chrome-no-host packages/*/test*.py
          cache-dir: .pytest_cache_chrome
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-chrome
          test-params: --runtime=chrome-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_chrome
          requires:
            - test-packages-chrome-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-firefox-no-numpy-dependents
          test-params: --runtime=firefox-no-host packages/*/test*.py
          cache-dir: .pytest_cache_firefox
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-firefox
          test-params: --runtime=firefox-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_firefox
          requires:
            - test-packages-firefox-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-node-no-numpy-dependents
          test-params: --runtime=node-no-host packages/*/test*.py
          cache-dir: .pytest_cache_node
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-node
          test-params: --runtime=node-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_node
          requires:
            - test-packages-node-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main-macos:
          name: test-packages-safari-no-numpy-dependents
          test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py
          cache-dir: .pytest_cache_safari
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main-macos:
          name: test-packages-safari
          test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_safari
          requires:
            - test-packages-safari-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-js:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark-stack-size:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - create-xbuild-env:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-cmdline-runner:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - build-pyodide-debug:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - deploy-release:
          requires:
            - test-core-firefox
            - test-packages-firefox
            - build-pyodide-debug
            - create-xbuild-env
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^\d+\.\d+\.\w+$/
          context:
            - s3-deployment
      - deploy-dev:
          requires:
            - test-core-firefox
            - test-packages-firefox
            - build-pyodide-debug
          filters:
            branches:
              only: main
          context:
            - s3-deployment

version: 2.1

defaults: &defaults
  working_directory: ~/repo
  docker:
    # Note: when updating the docker image version,
    #       make sure there are no extra old versions lying around.
    #       (e.g. `rg -F --hidden <old_tag>`)
    - image: pyodide/pyodide-env:20230301-chrome109-firefox109-py311
  environment:
    - EMSDK_NUM_CORES: 3
      EMCC_CORES: 3
      PYODIDE_JOBS: 3
      # Make sure the ccache dir is consistent between core and package builds
      # (it's not the case otherwise)
      CCACHE_DIR: /root/.ccache/

orbs:
  macos: circleci/macos@2.2.0

jobs:
  build-core:
    <<: *defaults
    steps:
      - checkout

      - restore_cache:
          keys:
            - -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
            - -core-v20220915-{{ checksum "cpython/Makefile" }}
            - -core-v20220915

      - run:
          name: calculate build hash for ccache
          command: |
            pip install pathspec
            ./tools/calculate_build_cache_key.py > build_hash.txt

      - run:
          name: build emsdk
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C emsdk
            ccache -s

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

      - run:
          name: build cpython
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C cpython
            ccache -s

      - run:
          name: build pyodide core
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            PYODIDE_PACKAGES="tag:core" make
            ccache -s

      - run:
          name: check-size
          command: |
            du dist/ -abh --max-depth 1  | sort -k 2

            pip install brotli
            ./tools/check_compressed_size.py dist/pyodide.asm.* dist/python_stdlib*

      - save_cache:
          paths:
            - /root/.ccache
          key: -core-v20220915-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}

      - run:
          name: Clean up workspace
          command: |
            rm -rf cpython/{build,downloads}
            rm -rf emsdk/emsdk/binaryen

      - persist_to_workspace:
          root: .
          paths:
            - .

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide.tar.gz dist
            tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm,_py.tar} dist/{package,repodata}.json dist/python_stdlib.zip

      - store_artifacts:
          path: /root/repo/dist/

      - store_artifacts:
          path: /root/repo/pyodide.tar.gz

      - store_artifacts:
          path: /root/repo/pyodide-core.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

  build-packages:
    parameters:
      packages:
        description: The packages to be built.
        type: string
    <<: *defaults
    resource_class: large
    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>

      - run:
          name: build packages
          no_output_timeout: 60m
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='<< parameters.packages >>' make dist/repodata.json
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: du dist/ -abh --max-depth 1  | sort -k 2

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide.tar.gz dist
            tar cjf pyodide-core.tar.gz dist/pyodide{.js,.mjs,.asm.js,.asm.wasm,_py.tar} dist/{package,repodata}.json dist/python_stdlib.zip
            tar cjf build-logs.tar.gz  packages/build-logs

      - run:
          name: Clean up package source files
          command: |
            cd packages && find **/build ! -name '.packaged' -type f -exec rm -f {} +

      - store_artifacts:
          path: /root/repo/pyodide.tar.gz

      - store_artifacts:
          path: /root/repo/pyodide-core.tar.gz

      - store_artifacts:
          path: /root/repo/build-logs.tar.gz

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg3-v20220915-{{ checksum "build_hash.txt" }}-<< parameters.packages >>

  build-pyodide-debug:
    <<: *defaults
    resource_class: large
    steps:
      - checkout

      - attach_workspace:
          at: .

      - run:
          name: build pyodide debug
          command: |
            cp -r dist dist-release
            rm dist/pyodide.asm.js
            source pyodide_env.sh
            ccache -z
            PYODIDE_DEBUG=1 make dist/pyodide.asm.js
            ccache -s
            cd dist
            npx prettier -w pyodide.asm.js
            npx prettier -w pyodide.js
            cd ..
            mv dist dist-debug
            mv dist-release dist

      - persist_to_workspace:
          root: .
          paths:
            - .

  create-xbuild-env:
    <<: *defaults
    steps:
      - checkout

      - attach_workspace:
          at: .

      - run:
          name: create xbuild environment
          command: |
            pip install -e ./pyodide-build
            python -m pyodide_build create_xbuildenv

      - run:
          name: Zip xbuild environment
          command: |
            tar cjf xbuildenv.tar.gz  xbuildenv/

      - store_artifacts:
          path: /root/repo/xbuildenv.tar.gz

      - persist_to_workspace:
          root: .
          paths:
            - ./xbuildenv

  test-main:
    parameters:
      test-params:
        description: The tests to run.
        type: string
      cache-dir:
        description: pytest-cache-dir.
        type: string
        default: ""
    <<: *defaults
    resource_class: medium+
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            make npm-link
            mkdir test-results
            pip install -r requirements.txt
            pip install -e ./pyodide-build
            npm install -g node-fetch@2
            if [ -z "<< parameters.cache-dir >>" ]; then
              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
            else
              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
            fi
            echo "pytest cache dir: $CACHE_DIR"
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 50 \
              --benchmark-json=benchmark-time.json \
              --benchmark-columns=mean,min,max,stddev \
              << parameters.test-params >> \
              -o cache_dir=$CACHE_DIR
      - store_test_results:
          path: test-results

  test-main-macos:
    parameters:
      test-params:
        description: The tests to run.
        type: string
      cache-dir:
        description: pytest-cache-dir.
        type: string
        default: ""
    resource_class: medium
    macos:
      xcode: 13.3.1

    working_directory: ~/repo
    steps:
      - attach_workspace:
          at: .
      # The standard way of enabling safaridriver no longer works for Safari 14+
      # https://blog.bytesguy.com/enabling-remote-automation-in-safari-14
      - macos/add-safari-permissions
      - run:
          name: install miniforge
          command: |
            curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
            bash Miniforge3-MacOSX-x86_64.sh -b && rm -f Miniforge3-MacOSX-x86_64.sh
            ~/miniforge3/bin/conda create -n pyodide python=3.11 -y
      - run:
          name: install dependencies
          command: |
            export PATH="$HOME/miniforge3/bin:$PATH"
            conda create -n pyodide python=3.11 -y
            source activate pyodide
            python -m pip install -r requirements.txt
            pip install -e ./pyodide-build
      - run:
          name: test safari
          command: |
            export PATH="$HOME/miniforge3/bin:$PATH"
            source activate pyodide
            mkdir test-results
            if [ -z "<< parameters.cache-dir >>" ]; then
              export CACHE_DIR=".test_cache/.pytest_cache_$(echo $RANDOM | md5sum | head -c 10)"
            else
              export CACHE_DIR=".test_cache/<< parameters.cache-dir >>"
            fi
            echo "pytest cache dir: $CACHE_DIR"
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 50 \
              << parameters.test-params >> \
              -o cache_dir=$CACHE_DIR
      - store_test_results:
          path: test-results

  benchmark-stack-size:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .
      - run:
          name: stack-size
          command: |
            make npm-link
            pip install -r requirements.txt
            npm install -g node-fetch@2
            pytest -s benchmark/stack_usage.py --rt node chrome firefox | sed -n 's/## //pg'

  test-js:
    <<: *defaults
    resource_class: small
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            cd src/test-js
            npm ci
            npm link ../../dist
            npm run test-types
            npm test
      # - run:
      #     name: check if Pyodide works with Webpack
      #     command: |
      #       git clone https://github.com/pyodide/pyodide-webpack-example.git
      #       export DEV_PYODIDE_PATH=`realpath dist`
      #       cd pyodide-webpack-example
      #       git checkout 164054a9c6fbd2176f386b6552ed8d079c6bcc04
      #       ./build.sh

      - run:
          name: test npm deploy (dry run)
          command: |
            DRY_RUN=1 ./tools/deploy_to_npm.sh

  test-cmdline-runner:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .

      - run:
          name: test
          command: |
            set -x
            export PYODIDE_ROOT=$(pwd)
            echo $PYODIDE_ROOT
            cd ~
            mkdir test
            cd test
            git clone https://github.com/python-attrs/attrs --depth 1 --branch 22.1.0
            git clone https://github.com/zopefoundation/zope.interface.git --depth 1 --branch 5.4.0

            python -m venv .venv-host
            source .venv-host/bin/activate
            pip install $PYODIDE_ROOT/pyodide-build
            pyodide venv .venv-pyodide
            source .venv-pyodide/bin/activate

            cd zope.interface
            pyodide build
            pip install dist/*.whl
            cd ../attrs
            pip install .[tests]
            python -m pytest -k 'not mypy'

  benchmark:
    <<: *defaults
    resource_class: medium+
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: install requirements
          command: |
            pip3 install numpy matplotlib pandas
            pip install -r requirements.txt
            npm install -g node-fetch@2
      - run:
          name: benchmark
          command: |
            python benchmark/benchmark.py all --output dist/benchmarks.json

      - store_artifacts:
          path: /root/repo/dist/benchmarks.json

  deploy-release:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-dev!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make npm
            python3 -m pip install awscli

      - run:
          name: Deploy Github Releases
          command: |
            mkdir -p /tmp/ghr/dist
            cp -r dist /tmp/ghr/pyodide
            cp -r xbuildenv /tmp/ghr/xbuildenv
            cd /tmp/ghr
            tar cjf dist/pyodide-${CIRCLE_TAG}.tar.bz2  pyodide/
            tar cjf dist/pyodide-core-${CIRCLE_TAG}.tar.bz2 pyodide/pyodide{.js,.mjs,.asm.js,.asm.wasm,_py.tar} pyodide/{package,repodata}.json pyodide/python_stdlib.zip
            tar cjf dist/xbuildenv-${CIRCLE_TAG}.tar.bz2  xbuildenv/
            ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" \
              -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" \
              -delete "${CIRCLE_TAG}" \
              dist

      - run:
          name: Deploy to npm
          command: |
            ./tools/deploy_to_npm.sh

      - run:
          name: Set PYODIDE_BASE_URL
          command: |
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/debug/" make dist/console.html
            cp dist/console.html dist-debug/console.html
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make dist/console.html
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip'    # 1 year cache
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year
      - run:
          name: Deploy debug version to pyodide-cdn2.iodide.io
          command: |
            find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*.data' --cache-control 'max-age=30758400, public' --content-encoding 'gzip'  # 1 year cache
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year cache
      - run:
          name: update 301 redirect for the /latest/* route.
          command: |
            aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json

  deploy-dev:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-release!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Set PYODIDE_BASE_URL
          command: |
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/debug/" make dist/console.html
            cp dist/console.html dist-debug/console.html
            PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make dist/console.html
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find dist/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync dist/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache
      - run:
          name: Deploy debug version to pyodide-cdn2.iodide.io
          command: |
            find dist-debug/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/debug/"
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync dist-debug/ "s3://pyodide-cdn2.iodide.io/dev/debug/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache

workflows:
  version: 2
  build-and-deploy:
    jobs:
      - build-core:
          filters:
            tags:
              only: /.*/

      - build-packages:
          name: build-packages-no-numpy-dependents
          packages: "*,no-numpy-dependents"
          requires:
            - build-core
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages
                  - ./dist

      - build-packages:
          name: build-packages-opencv-python
          packages: opencv-python
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages/opencv-python/build
                  - ./packages/opencv-python/dist
                  - ./packages/build-logs/opencv*
                  - ./dist/opencv*

      - build-packages:
          name: build-packages-numpy-dependents
          packages: "*,!opencv-python"
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages
                  - ./dist

      - build-packages:
          name: build-packages
          packages: "*"
          requires:
            - build-packages-numpy-dependents
            - build-packages-opencv-python
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./packages/.artifacts
                  - ./dist

      - test-main:
          name: test-core-chrome
          test-params: --runtime=chrome-no-host -k "not webworker" src packages/micropip  packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox
          test-params: --runtime=firefox-no-host -k "not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-node
          test-params: --runtime=node-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/ pyodide-build/pyodide_build/tests
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main-macos:
          name: test-core-safari
          test-params: --runtime=safari-no-host src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome-webworker
          test-params: --runtime=chrome-no-host src/tests/test_webworker.py
          requires:
            - test-core-chrome
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox-webworker
          test-params: --runtime=firefox-no-host src/tests/test_webworker.py
          requires:
            - test-core-firefox
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-chrome-no-numpy-dependents
          test-params: --runtime=chrome-no-host packages/*/test*.py
          cache-dir: .pytest_cache_chrome
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-chrome
          test-params: --runtime=chrome-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_chrome
          requires:
            - test-packages-chrome-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-firefox-no-numpy-dependents
          test-params: --runtime=firefox-no-host packages/*/test*.py
          cache-dir: .pytest_cache_firefox
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-firefox
          test-params: --runtime=firefox-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_firefox
          requires:
            - test-packages-firefox-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-node-no-numpy-dependents
          test-params: --runtime=node-no-host packages/*/test*.py
          cache-dir: .pytest_cache_node
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main:
          name: test-packages-node
          test-params: --runtime=node-no-host packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_node
          requires:
            - test-packages-node-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main-macos:
          name: test-packages-safari-no-numpy-dependents
          test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py
          cache-dir: .pytest_cache_safari
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/
          post-steps:
            - persist_to_workspace:
                root: .
                paths:
                  - ./.test_cache

      - test-main-macos:
          name: test-packages-safari
          test-params: --runtime=safari-no-host -k "not webworker" packages/*/test*.py --skip-passed
          cache-dir: .pytest_cache_safari
          requires:
            - test-packages-safari-no-numpy-dependents
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-js:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark-stack-size:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - create-xbuild-env:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-cmdline-runner:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - build-pyodide-debug:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - deploy-release:
          requires:
            - test-core-firefox
            - test-packages-firefox
            - build-pyodide-debug
            - create-xbuild-env
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^\d+\.\d+\.\w+$/
          context:
            - s3-deployment
      - deploy-dev:
          requires:
            - test-core-firefox
            - test-packages-firefox
            - build-pyodide-debug
          filters:
            branches:
              only: main
          context:
            - s3-deployment
