Instead of trying to convince setup.py to do what we want, the wheel
package empowers us to unpack and directly manipulate the wheel.
In particular this is important because `--skip-build` really does
not work.

First we build a native wheel. If skip_host is True, then this
"native" wheel has defective empty files as binaries, if it's False,
we build a real native wheel. If skip_host is False, then we use
pip to install our native wheel into host directory.

Then we unpack the wheel, replace the platform tag with
wasm32_emscripten, remove the non-metadata files, and copy in the
new files with the cross compiled binaries.

Of course, this is all only necessary for binary extensions.


Hopefully this will help get us towards using alternative build
backends. The one concern is that this approach depends on
implementation details of where the build backend puts the files
into the directory tree before copying them into the wheel. If
the build backend moved the linked binaries after building them,
we would be in trouble. Or worse if the build backend deleted the
files after packing them into the wheel, we'd be in trouble.

Basically, it seems clear that we can only support alternative
build backends on a case-by-case basis.

@henryiii I'm curious what your thoughts are cross compilation + arbitrary build backends.

Better package cross compilation. Instead of trying to convince setup.py to do what we want, the wheel
package empowers us to unpack and directly manipulate the wheel.
In particular this is important because `--skip-build` really does
not work.

First we build a native wheel. If skip_host is True, then this
"native" wheel has defective empty files as binaries, if it's False,
we build a real native wheel. If skip_host is False, then we use
pip to install our native wheel into host directory.

Then we unpack the wheel, replace the platform tag with
wasm32_emscripten, remove the non-metadata files, and copy in the
new files with the cross compiled binaries.

Of course, this is all only necessary for binary extensions.


Hopefully this will help get us towards using alternative build
backends. The one concern is that this approach depends on
implementation details of where the build backend puts the files
into the directory tree before copying them into the wheel. If
the build backend moved the linked binaries after building them,
we would be in trouble. Or worse if the build backend deleted the
files after packing them into the wheel, we'd be in trouble.

Basically, it seems clear that we can only support alternative
build backends on a case-by-case basis.

@henryiii I'm curious what your thoughts are cross compilation + arbitrary build backends.

version: 2.1

defaults: &defaults
  working_directory: ~/repo
  docker:
    # Note: when updating the docker image version,
    #       make sure there are no extra old versions lying around.
    #       (e.g. `rg -F --hidden <old_tag>`)
    - image: pyodide/pyodide-env:20220213-chrome97-firefox96
  environment:
    - EMSDK_NUM_CORES: 3
      EMCC_CORES: 3
      PYODIDE_JOBS: 3
      # Make sure the ccache dir is consistent between core and package builds
      # (it's not the case otherwise)
      CCACHE_DIR: /root/.ccache/

jobs:
  test-docs:
    <<: *defaults
    resource_class: small
    steps:
      - checkout

      - run:
          name: Test docs
          command: |
            mkdir test-results
            pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml
      - store_test_results:
          path: test-results

  build-core:
    <<: *defaults
    steps:
      - checkout

      - restore_cache:
          keys:
            - -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
            - -core-v20220303-{{ checksum "cpython/Makefile" }}
            - -core-v20220303

      - run:
          name: build emsdk
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C emsdk
            ccache -s

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

      - run:
          name: build cpython
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C cpython
            ccache -s

      - run:
          name: build pyodide core
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            PYODIDE_PACKAGES="core" make
            ccache -s

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}

      - run:
          name: Clean up workspace
          command: |
            rm -rf cpython/{build,downloads}
            rm -rf emsdk/emsdk/binaryen

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide-build.tar.gz  build

      - persist_to_workspace:
          root: .
          paths:
            - .

      - store_artifacts:
          path: /root/repo/build/

      - store_artifacts:
          path: /root/repo/pyodide-build.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

  build-packages-no-numpy-dependents:
    <<: *defaults
    resource_class: large

    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg1-v20220303-{{ checksum "Makefile.envs" }}
            - -pkg1-v20220303

      - run:
          name: build packages
          no_output_timeout: 30m
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='*, no-numpy-dependents' make -C packages
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg1-v20220303-{{ checksum "Makefile.envs" }}

      - persist_to_workspace:
          root: .
          paths:
            - ./packages
            - ./build

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide-build.tar.gz  build
            tar cjf build-logs.tar.gz  packages/build-logs

      - persist_to_workspace:
          root: .
          paths:
            - .

      - store_artifacts:
          path: /root/repo/build/

      - store_artifacts:
          path: /root/repo/pyodide-build.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

      - store_artifacts:
          path: /root/repo/build-logs.tar.gz

  build-packages:
    <<: *defaults
    resource_class: large

    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg2-v20220303-{{ checksum "Makefile.envs" }}
            - -pkg2-v20220303

      - run:
          name: build packages
          no_output_timeout: 30m
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='*' make -C packages
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg2-v20220303-{{ checksum "Makefile.envs" }}

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide-build.tar.gz  build
            tar cjf build-logs.tar.gz  packages/build-logs

      - persist_to_workspace:
          root: .
          paths:
            - ./packages/.artifacts
            - ./build

      - store_artifacts:
          path: /root/repo/build/

      - store_artifacts:
          path: /root/repo/pyodide-build.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

      - store_artifacts:
          path: /root/repo/build-logs.tar.gz

  test-main:
    parameters:
      test-params:
        description: The tests to run.
        type: string
    <<: *defaults
    resource_class: medium+
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            mkdir test-results
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 10 \
              << parameters.test-params >>
      - store_test_results:
          path: test-results

  benchmark-stack-size:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .
      - run:
          name: stack-size
          command: |
            pytest -s benchmark/stack_usage.py  | sed -n 's/## //pg'

  test-python:
    # Unlike test-main this uses "resource_class small" and doesn't use "pytest_wrapper."
    <<: *defaults
    resource_class: small
    steps:
      - checkout
      - run:
          name: test
          command: |
            mkdir test-results
            pytest \
              --junitxml=test-results/junit.xml \
              --verbose \
              -k 'not (chrome or firefox or node)' \
              --cov=pyodide_build --cov=pyodide \
              src pyodide-build packages/micropip/

      - store_test_results:
          path: test-results

  test-js:
    <<: *defaults
    resource_class: small
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            cd src/js
            npx tsd
            npm ci
            npm test
      # - run:
      #     name: check if webpack cli works well with load-pyodide.js
      #     command: |
      #       git clone https://github.com/pyodide/pyodide-webpack-example.git
      #       cd pyodide-webpack-example
      #       npm ci
      #       cp ../src/js/*.js node_modules/pyodide/
      #       npx webpack

  benchmark:
    <<: *defaults
    resource_class: medium+
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: benchmark
          command: |
            python benchmark/benchmark.py all --output build/benchmarks.json

      - store_artifacts:
          path: /root/repo/build/benchmarks.json

  deploy-release:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-dev!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Deploy Github Releases
          command: |
            cp -r build /tmp/pyodide
            cd /tmp/
            tar cjf pyodide-build-${CIRCLE_TAG}.tar.bz2  pyodide/
            ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" -delete  "${CIRCLE_TAG}" ./pyodide-build-${CIRCLE_TAG}.tar.bz2

      - run:
          name: Set PYODIDE_BASE_URL
          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make update_base_url
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip'    # 1 year cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year
            # update 301 redirect for the /latest/* route.
            aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json

  deploy-dev:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-release!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Set PYODIDE_BASE_URL
          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make update_base_url
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache

workflows:
  version: 2
  build-and-deploy:
    jobs:
      - test-docs:
          filters:
            tags:
              only: /.*/

      - test-python:
          filters:
            tags:
              only: /.*/

      - build-core:
          filters:
            tags:
              only: /.*/

      - build-packages-no-numpy-dependents:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - build-packages:
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome
          test-params: -k "chrome and not webworker" src packages/micropip  packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox
          test-params: -k "firefox and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-node
          test-params: -k node src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome-webworker
          test-params: -k chrome src/tests/test_webworker.py
          requires:
            - test-core-chrome
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox-webworker
          test-params: -k firefox src/tests/test_webworker.py
          requires:
            - test-core-firefox
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-chrome
          test-params: -k chrome packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-firefox
          test-params: -k firefox packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-node
          test-params: -k "node and not numpy" packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-js:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark-stack-size:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - deploy-release:
          requires:
            - test-python
            - test-core-firefox
            - test-packages-firefox
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^\d+\.\d+\.\w+$/
      - deploy-dev:
          requires:
            - test-python
            - test-core-firefox
            - test-packages-firefox
          filters:
            branches:
              only: main

version: 2.1

defaults: &defaults
  working_directory: ~/repo
  docker:
    # Note: when updating the docker image version,
    #       make sure there are no extra old versions lying around.
    #       (e.g. `rg -F --hidden <old_tag>`)
    - image: pyodide/pyodide-env:20220213-chrome97-firefox96
  environment:
    - EMSDK_NUM_CORES: 3
      EMCC_CORES: 3
      PYODIDE_JOBS: 3
      # Make sure the ccache dir is consistent between core and package builds
      # (it's not the case otherwise)
      CCACHE_DIR: /root/.ccache/

jobs:
  test-docs:
    <<: *defaults
    resource_class: small
    steps:
      - checkout

      - run:
          name: Test docs
          command: |
            mkdir test-results
            pytest docs/sphinx_pyodide/tests --junitxml=test-results/junit.xml
      - store_test_results:
          path: test-results

  build-core:
    <<: *defaults
    steps:
      - checkout

      - restore_cache:
          keys:
            - -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}
            - -core-v20220303-{{ checksum "cpython/Makefile" }}
            - -core-v20220303

      - run:
          name: build emsdk
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C emsdk
            ccache -s

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

      - run:
          name: build cpython
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            make -C cpython
            ccache -s

      - run:
          name: build pyodide core
          no_output_timeout: 20m
          command: |
            # This is necessary to use the ccache from emsdk
            source pyodide_env.sh

            ccache -z
            PYODIDE_PACKAGES="core" make
            ccache -s

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -core-v20220303-{{ checksum "cpython/Makefile" }}-{{ checksum "Makefile.envs" }}

      - run:
          name: Clean up workspace
          command: |
            rm -rf cpython/{build,downloads}
            rm -rf emsdk/emsdk/binaryen

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide-build.tar.gz  build

      - persist_to_workspace:
          root: .
          paths:
            - .

      - store_artifacts:
          path: /root/repo/build/

      - store_artifacts:
          path: /root/repo/pyodide-build.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

  build-packages-no-numpy-dependents:
    <<: *defaults
    resource_class: large

    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg1-v20220303-{{ checksum "Makefile.envs" }}
            - -pkg1-v20220303

      - run:
          name: build packages
          no_output_timeout: 30m
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='*, no-numpy-dependents' make -C packages
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg1-v20220303-{{ checksum "Makefile.envs" }}

      - persist_to_workspace:
          root: .
          paths:
            - ./packages
            - ./build

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide-build.tar.gz  build
            tar cjf build-logs.tar.gz  packages/build-logs

      - persist_to_workspace:
          root: .
          paths:
            - .

      - store_artifacts:
          path: /root/repo/build/

      - store_artifacts:
          path: /root/repo/pyodide-build.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

      - store_artifacts:
          path: /root/repo/build-logs.tar.gz

  build-packages:
    <<: *defaults
    resource_class: large

    steps:
      - checkout

      - attach_workspace:
          at: .

      - restore_cache:
          keys:
            - -pkg2-v20220303-{{ checksum "Makefile.envs" }}
            - -pkg2-v20220303

      - run:
          name: build packages
          no_output_timeout: 30m
          command: |
            source pyodide_env.sh

            # Set mtime for EM_CONFIG to avoid ccache cache misses
            touch -m -d '1 Jan 2021 12:00' emsdk/emsdk/.emscripten

            ccache -z
            PYODIDE_PACKAGES='*' make -C packages
            ccache -s
          environment:
            PYODIDE_JOBS: 5

      - run:
          name: check-size
          command: ls -lh build/

      - save_cache:
          paths:
            - /root/.ccache
          key: -pkg2-v20220303-{{ checksum "Makefile.envs" }}

      - run:
          name: Zip build directory
          command: |
            tar cjf pyodide-build.tar.gz  build
            tar cjf build-logs.tar.gz  packages/build-logs

      - persist_to_workspace:
          root: .
          paths:
            - ./packages/.artifacts
            - ./build

      - store_artifacts:
          path: /root/repo/build/

      - store_artifacts:
          path: /root/repo/pyodide-build.tar.gz

      - store_artifacts:
          path: /root/repo/packages/build-logs

      - store_artifacts:
          path: /root/repo/build-logs.tar.gz

  test-main:
    parameters:
      test-params:
        description: The tests to run.
        type: string
    <<: *defaults
    resource_class: medium+
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            mkdir test-results
            tools/pytest_wrapper.py \
              --junitxml=test-results/junit.xml \
              --verbose \
              --durations 10 \
              << parameters.test-params >>
      - store_test_results:
          path: test-results

  benchmark-stack-size:
    <<: *defaults
    steps:
      - attach_workspace:
          at: .
      - run:
          name: stack-size
          command: |
            pytest -s benchmark/stack_usage.py  | sed -n 's/## //pg'

  test-python:
    # Unlike test-main this uses "resource_class small" and doesn't use "pytest_wrapper."
    <<: *defaults
    resource_class: small
    steps:
      - checkout
      - run:
          name: test
          command: |
            mkdir test-results
            pytest \
              --junitxml=test-results/junit.xml \
              --verbose \
              -k 'not (chrome or firefox or node)' \
              --cov=pyodide_build --cov=pyodide \
              src pyodide-build packages/micropip/

      - store_test_results:
          path: test-results

  test-js:
    <<: *defaults
    resource_class: small
    steps:
      - attach_workspace:
          at: .
      - run:
          name: test
          command: |
            cd src/js
            npx tsd
            npm ci
            npm test
      # - run:
      #     name: check if webpack cli works well with load-pyodide.js
      #     command: |
      #       git clone https://github.com/pyodide/pyodide-webpack-example.git
      #       cd pyodide-webpack-example
      #       npm ci
      #       cp ../src/js/*.js node_modules/pyodide/
      #       npx webpack

  benchmark:
    <<: *defaults
    resource_class: medium+
    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: install requirements
          command: |
            pip3 install numpy matplotlib
      - run:
          name: benchmark
          command: |
            python benchmark/benchmark.py all --output build/benchmarks.json

      - store_artifacts:
          path: /root/repo/build/benchmarks.json

  deploy-release:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-dev!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Deploy Github Releases
          command: |
            cp -r build /tmp/pyodide
            cd /tmp/
            tar cjf pyodide-build-${CIRCLE_TAG}.tar.bz2  pyodide/
            ghr -t "${GITHUB_TOKEN}" -u "${CIRCLE_PROJECT_USERNAME}" -r "${CIRCLE_PROJECT_REPONAME}" -c "${CIRCLE_SHA1}" -delete  "${CIRCLE_TAG}" ./pyodide-build-${CIRCLE_TAG}.tar.bz2

      - run:
          name: Set PYODIDE_BASE_URL
          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/v${CIRCLE_TAG}/full/" make update_base_url
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*.data' --cache-control 'max-age=30758400, immutable, public' --content-encoding 'gzip'    # 1 year cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/v${CIRCLE_TAG}/full/" --exclude '*' --include '*.data' --cache-control 'max-age=30758400, immutable, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 year
            # update 301 redirect for the /latest/* route.
            aws s3api put-bucket-website --cli-input-json file://.circleci/s3-website-config.json

  deploy-dev:
    # To reduce chance of deployment issues, try to keep the steps here as
    # similar as possible to the steps in deploy-release!
    resource_class: small

    docker:
      - image: cibuilds/github:0.13

    steps:
      - checkout
      - attach_workspace:
          at: .
      - run:
          name: Install requirements
          command: |
            apk add --no-cache --update python3 make
            python3 -m pip install awscli
      - run:
          name: Set PYODIDE_BASE_URL
          command: PYODIDE_BASE_URL="https://cdn.jsdelivr.net/pyodide/dev/full/" make update_base_url
      - run:
          name: Deploy to pyodide-cdn2.iodide.io
          command: |
            find build/ -type f -print0 | xargs -0 -n1 -I@ bash -c "echo \"Compressing @\"; gzip @; mv @.gz @;"
            aws s3 rm --recursive "s3://pyodide-cdn2.iodide.io/dev/full/"
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*.data' --cache-control 'max-age=3600, public' --content-encoding 'gzip'  # 1 hour cache
            aws s3 sync build/ "s3://pyodide-cdn2.iodide.io/dev/full/" --exclude '*' --include '*.data' --cache-control 'max-age=3600, public'  --content-type 'application/wasm' --content-encoding 'gzip'  # 1 hour cache

workflows:
  version: 2
  build-and-deploy:
    jobs:
      - test-docs:
          filters:
            tags:
              only: /.*/

      - test-python:
          filters:
            tags:
              only: /.*/

      - build-core:
          filters:
            tags:
              only: /.*/

      - build-packages-no-numpy-dependents:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - build-packages:
          requires:
            - build-packages-no-numpy-dependents
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome
          test-params: -k "chrome and not webworker" src packages/micropip  packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox
          test-params: -k "firefox and not webworker" src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-node
          test-params: -k node src packages/micropip packages/fpcast-test packages/sharedlib-test-py/ packages/cpp-exceptions-test/
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-chrome-webworker
          test-params: -k chrome src/tests/test_webworker.py
          requires:
            - test-core-chrome
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-core-firefox-webworker
          test-params: -k firefox src/tests/test_webworker.py
          requires:
            - test-core-firefox
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-chrome
          test-params: -k chrome packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-firefox
          test-params: -k firefox packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-main:
          name: test-packages-node
          test-params: -k "node and not numpy" packages/test* packages/*/test*
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - test-js:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark-stack-size:
          requires:
            - build-core
          filters:
            tags:
              only: /.*/

      - benchmark:
          requires:
            - build-packages
          filters:
            tags:
              only: /.*/

      - deploy-release:
          requires:
            - test-python
            - test-core-firefox
            - test-packages-firefox
          filters:
            branches:
              ignore: /.*/
            tags:
              only: /^\d+\.\d+\.\w+$/
      - deploy-dev:
          requires:
            - test-python
            - test-core-firefox
            - test-packages-firefox
          filters:
            branches:
              only: main
