<!-- Thank you for contributing to Pyodide! All improvements are welcome,
     so don't be afraid to make a PR. -->

<!-- [IMPORTANT] Note on CI failures:
     Currently, we are having issues with selenium-based tests.
     Don't panic if the CI fails on your PR because of timeouts.
     It's probably not your fault. We will investigate :) -->

### Description

<!-- Please explain what your PR is about:
     - reasoning for the change
     - some details of updated code
     - any noteworthy choices to be aware of
	Please refer to any related issues by #<issue_id> -->

Working on using pre-commit properly. Will be a series of commits, and undrafted after they are complete.

Rough plan:

- [x] Commit 1: update pre-commit, prepare mypy to run correctly.
- [x] Next commit: apply pre-commit fixes
- [x] Next commit: use pre-commit instead of manual runs inside Makefile.
- [x] Somewhere in between commit: fix manual issues

### Checklists

<!-- Note on checklists:
     If you think some of these steps are not necessary for your PR,
     just remove those checkboxes, or mark them as checked. Otherwise,
     if some checkboxes are left unmarked, we might assume that your PR
     is not ready to be merged and we might keep you waiting -->

style: improve pre-commit. <!-- Thank you for contributing to Pyodide! All improvements are welcome,
     so don't be afraid to make a PR. -->

<!-- [IMPORTANT] Note on CI failures:
     Currently, we are having issues with selenium-based tests.
     Don't panic if the CI fails on your PR because of timeouts.
     It's probably not your fault. We will investigate :) -->

### Description

<!-- Please explain what your PR is about:
     - reasoning for the change
     - some details of updated code
     - any noteworthy choices to be aware of
	Please refer to any related issues by #<issue_id> -->

Working on using pre-commit properly. Will be a series of commits, and undrafted after they are complete.

Rough plan:

- [x] Commit 1: update pre-commit, prepare mypy to run correctly.
- [x] Next commit: apply pre-commit fixes
- [x] Next commit: use pre-commit instead of manual runs inside Makefile.
- [x] Somewhere in between commit: fix manual issues

### Checklists

<!-- Note on checklists:
     If you think some of these steps are not necessary for your PR,
     just remove those checkboxes, or mark them as checked. Otherwise,
     if some checkboxes are left unmarked, we might assume that your PR
     is not ready to be merged and we might keep you waiting -->

PYODIDE_ROOT=$(abspath .)

include Makefile.envs

.PHONY=check

CC=emcc
CXX=em++


all: check \
	build/pyodide.asm.js \
	build/pyodide.js \
	build/console.html \
	build/distutils.tar \
	build/packages.json \
	build/pyodide_py.tar \
	build/test.tar \
	build/test.html \
	build/webworker.js \
	build/webworker_dev.js
	echo -e "\nSUCCESS!"

$(CPYTHONLIB)/tzdata :
	pip install tzdata --target=$(CPYTHONLIB)

build/pyodide_py.tar: $(wildcard src/py/pyodide/*.py)  $(wildcard src/py/_pyodide/*.py)
	cd src/py && tar --exclude '*__pycache__*' -cf ../../build/pyodide_py.tar pyodide _pyodide

build/pyodide.asm.js: \
	src/core/docstring.o \
	src/core/error_handling.o \
	src/core/error_handling_cpp.o \
	src/core/numpy_patch.o \
	src/core/hiwire.o \
	src/core/js2python.o \
	src/core/jsproxy.o \
	src/core/keyboard_interrupt.o \
	src/core/main.o  \
	src/core/pyproxy.o \
	src/core/python2js_buffer.o \
	src/core/python2js.o \
	$(wildcard src/py/lib/*.py) \
	$(CPYTHONLIB)/tzdata \
	$(CPYTHONLIB)
	date +"[%F %T] Building pyodide.asm.js..."
	[ -d build ] || mkdir build
	$(CXX) -o build/pyodide.asm.js $(filter %.o,$^) \
		$(MAIN_MODULE_LDFLAGS)
   # Strip out C++ symbols which all start __Z.
   # There are 4821 of these and they have VERY VERY long names.
   # To show some stats on the symbols you can use the following:
   # cat build/pyodide.asm.js | grep -ohE 'var _{0,5}.' | sort | uniq -c | sort -nr | head -n 20
	sed -i -E 's/var __Z[^;]*;//g' build/pyodide.asm.js
	sed -i '1i\
		"use strict";\
		let setImmediate = globalThis.setImmediate;\
		let clearImmediate = globalThis.clearImmediate;\
		let baseName, fpcGOT, dyncallGOT, fpVal, dcVal;\
	' build/pyodide.asm.js
	echo "globalThis._createPyodideModule = _createPyodideModule;" >> build/pyodide.asm.js
	date +"[%F %T] done building pyodide.asm.js."


env:
	env


node_modules/.installed : src/js/package.json src/js/package-lock.json
	cd src/js && npm ci
	ln -sfn src/js/node_modules/ node_modules
	touch node_modules/.installed

build/pyodide.js: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts node_modules/.installed
	npx rollup -c src/js/rollup.config.js

src/js/error_handling.gen.ts : src/core/error_handling.ts
	cp $< $@

src/js/pyproxy.gen.ts : src/core/pyproxy.* src/core/*.h
	# We can't input pyproxy.js directly because CC will be unhappy about the file
	# extension. Instead cat it and have CC read from stdin.
	# -E : Only apply prepreocessor
	# -C : Leave comments alone (this allows them to be preserved in typescript
	#      definition files, rollup will strip them out)
	# -P : Don't put in macro debug info
	# -imacros pyproxy.c : include all of the macros definitions from pyproxy.c
	#
	# First we use sed to delete the segments of the file between
	# "// pyodide-skip" and "// end-pyodide-skip". This allows us to give
	# typescript type declarations for the macros which we need for intellisense
	# and documentation generation. The result of processing the type
	# declarations with the macro processor is a type error, so we snip them
	# out.
	rm -f $@
	echo "// This file is generated by applying the C preprocessor to core/pyproxy.ts" >> $@
	echo "// It uses the macros defined in core/pyproxy.c" >> $@
	echo "// Do not edit it directly!" >> $@
	cat src/core/pyproxy.ts | \
		sed '/^\/\/\s*pyodide-skip/,/^\/\/\s*end-pyodide-skip/d' | \
		$(CC) -E -C -P -imacros src/core/pyproxy.c $(MAIN_MODULE_CFLAGS) - \
		>> $@

build/test.html: src/templates/test.html
	cp $< $@


.PHONY: build/console.html
build/console.html: src/templates/console.html
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@


.PHONY: docs/_build/html/console.html
docs/_build/html/console.html: src/templates/console.html
	mkdir -p docs/_build/html
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@


.PHONY: build/webworker.js
build/webworker.js: src/templates/webworker.js
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@


.PHONY: build/webworker_dev.js
build/webworker_dev.js: src/templates/webworker.js
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#./#g' $@


update_base_url: \
	build/console.html \
	build/webworker.js



lint: node_modules/.installed
	# check for unused imports, the rest is done by black
	flake8 --select=F401 src tools pyodide-build benchmark conftest.py docs packages/matplotlib/src/
	mypy --ignore-missing-imports    \
		pyodide-build/pyodide_build/ \
		src/ 					     \
		packages/*/test* 			 \
		conftest.py 				 \
		docs
#mypy gets upset about there being both : src / py / setup.py and
#packages / micropip / src / setup.py.There is no easy way to fix this right now
#see python / mypy #10428. This will also cause trouble with pre - commit if you
#modify both setup.py files in the same commit.
	mypy --ignore-missing-imports    \
		packages/micropip/src/

	# Format checks
	for file in src/core/*.c src/core/*.h ; do \
		clang-format-6.0 -output-replacements-xml $$file | grep '<replacement ' > /dev/null ; \
		if [ $$? -eq 0 ] ; then \
			echo clang-format errors for $$file ; \
			exit 1 ; \
		fi ; \
	done
	npx prettier --check .
	black --check .


benchmark: all
	$(HOSTPYTHON) benchmark/benchmark.py $(HOSTPYTHON) build/benchmarks.json
	$(HOSTPYTHON) benchmark/plot_benchmark.py build/benchmarks.json build/benchmarks.png


clean:
	rm -fr build/*
	rm -fr src/*/*.o
	rm -fr node_modules
	make -C packages clean
	echo "The Emsdk, CPython are not cleaned. cd into those directories to do so."

clean-python: clean
	make -C cpython clean

clean-all:
	make -C emsdk clean
	make -C cpython clean-all

src/core/error_handling_cpp.o: src/core/error_handling_cpp.cpp
	$(CXX) -o $@ -c $< $(MAIN_MODULE_CFLAGS) -Isrc/core/

%.o: %.c $(CPYTHONLIB) $(wildcard src/core/*.h src/core/*.js)
	$(CC) -o $@ -c $< $(MAIN_MODULE_CFLAGS) -Isrc/core/


# Stdlib modules that we repackage as standalone packages

# TODO: also include test directories included in other stdlib modules
build/test.tar: $(CPYTHONLIB) node_modules/.installed
	cd $(CPYTHONLIB) && tar --exclude=__pycache__ -cf $(PYODIDE_ROOT)/build/test.tar test

build/distutils.tar: $(CPYTHONLIB) node_modules/.installed
	cd $(CPYTHONLIB) && tar --exclude=__pycache__ -cf $(PYODIDE_ROOT)/build/distutils.tar distutils


$(CPYTHONLIB): emsdk/emsdk/.complete $(PYODIDE_EMCC) $(PYODIDE_CXX)
	date +"[%F %T] Building cpython..."
	make -C $(CPYTHONROOT)
	date +"[%F %T] done building cpython..."


build/packages.json: FORCE
	date +"[%F %T] Building packages..."
	make -C packages
	date +"[%F %T] done building packages..."


emsdk/emsdk/.complete:
	date +"[%F %T] Building emsdk..."
	make -C emsdk
	date +"[%F %T] done building emsdk."


FORCE:


check:
	./tools/dependency-check.sh


debug :
	EXTRA_CFLAGS+=" -D DEBUG_F" \
	make

PYODIDE_ROOT=$(abspath .)

include Makefile.envs

.PHONY=check

CC=emcc
CXX=em++


all: check \
	build/pyodide.asm.js \
	build/pyodide.js \
	build/console.html \
	build/distutils.tar \
	build/packages.json \
	build/pyodide_py.tar \
	build/test.tar \
	build/test.html \
	build/webworker.js \
	build/webworker_dev.js
	echo -e "\nSUCCESS!"

$(CPYTHONLIB)/tzdata :
	pip install tzdata --target=$(CPYTHONLIB)

build/pyodide_py.tar: $(wildcard src/py/pyodide/*.py)  $(wildcard src/py/_pyodide/*.py)
	cd src/py && tar --exclude '*__pycache__*' -cf ../../build/pyodide_py.tar pyodide _pyodide

build/pyodide.asm.js: \
	src/core/docstring.o \
	src/core/error_handling.o \
	src/core/error_handling_cpp.o \
	src/core/numpy_patch.o \
	src/core/hiwire.o \
	src/core/js2python.o \
	src/core/jsproxy.o \
	src/core/keyboard_interrupt.o \
	src/core/main.o  \
	src/core/pyproxy.o \
	src/core/python2js_buffer.o \
	src/core/python2js.o \
	$(wildcard src/py/lib/*.py) \
	$(CPYTHONLIB)/tzdata \
	$(CPYTHONLIB)
	date +"[%F %T] Building pyodide.asm.js..."
	[ -d build ] || mkdir build
	$(CXX) -o build/pyodide.asm.js $(filter %.o,$^) \
		$(MAIN_MODULE_LDFLAGS)
   # Strip out C++ symbols which all start __Z.
   # There are 4821 of these and they have VERY VERY long names.
   # To show some stats on the symbols you can use the following:
   # cat build/pyodide.asm.js | grep -ohE 'var _{0,5}.' | sort | uniq -c | sort -nr | head -n 20
	sed -i -E 's/var __Z[^;]*;//g' build/pyodide.asm.js
	sed -i '1i\
		"use strict";\
		let setImmediate = globalThis.setImmediate;\
		let clearImmediate = globalThis.clearImmediate;\
		let baseName, fpcGOT, dyncallGOT, fpVal, dcVal;\
	' build/pyodide.asm.js
	echo "globalThis._createPyodideModule = _createPyodideModule;" >> build/pyodide.asm.js
	date +"[%F %T] done building pyodide.asm.js."


env:
	env


node_modules/.installed : src/js/package.json src/js/package-lock.json
	cd src/js && npm ci
	ln -sfn src/js/node_modules/ node_modules
	touch node_modules/.installed

build/pyodide.js: src/js/*.ts src/js/pyproxy.gen.ts src/js/error_handling.gen.ts node_modules/.installed
	npx rollup -c src/js/rollup.config.js

src/js/error_handling.gen.ts : src/core/error_handling.ts
	cp $< $@

src/js/pyproxy.gen.ts : src/core/pyproxy.* src/core/*.h
	# We can't input pyproxy.js directly because CC will be unhappy about the file
	# extension. Instead cat it and have CC read from stdin.
	# -E : Only apply prepreocessor
	# -C : Leave comments alone (this allows them to be preserved in typescript
	#      definition files, rollup will strip them out)
	# -P : Don't put in macro debug info
	# -imacros pyproxy.c : include all of the macros definitions from pyproxy.c
	#
	# First we use sed to delete the segments of the file between
	# "// pyodide-skip" and "// end-pyodide-skip". This allows us to give
	# typescript type declarations for the macros which we need for intellisense
	# and documentation generation. The result of processing the type
	# declarations with the macro processor is a type error, so we snip them
	# out.
	rm -f $@
	echo "// This file is generated by applying the C preprocessor to core/pyproxy.ts" >> $@
	echo "// It uses the macros defined in core/pyproxy.c" >> $@
	echo "// Do not edit it directly!" >> $@
	cat src/core/pyproxy.ts | \
		sed '/^\/\/\s*pyodide-skip/,/^\/\/\s*end-pyodide-skip/d' | \
		$(CC) -E -C -P -imacros src/core/pyproxy.c $(MAIN_MODULE_CFLAGS) - \
		>> $@

build/test.html: src/templates/test.html
	cp $< $@


.PHONY: build/console.html
build/console.html: src/templates/console.html
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@


.PHONY: docs/_build/html/console.html
docs/_build/html/console.html: src/templates/console.html
	mkdir -p docs/_build/html
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@


.PHONY: build/webworker.js
build/webworker.js: src/templates/webworker.js
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#$(PYODIDE_BASE_URL)#g' $@


.PHONY: build/webworker_dev.js
build/webworker_dev.js: src/templates/webworker.js
	cp $< $@
	sed -i -e 's#{{ PYODIDE_BASE_URL }}#./#g' $@


update_base_url: \
	build/console.html \
	build/webworker.js



.PHONY: lint
lint:
	pre-commit run -a --show-diff-on-failure

benchmark: all
	$(HOSTPYTHON) benchmark/benchmark.py $(HOSTPYTHON) build/benchmarks.json
	$(HOSTPYTHON) benchmark/plot_benchmark.py build/benchmarks.json build/benchmarks.png


clean:
	rm -fr build/*
	rm -fr src/*/*.o
	rm -fr node_modules
	make -C packages clean
	echo "The Emsdk, CPython are not cleaned. cd into those directories to do so."

clean-python: clean
	make -C cpython clean

clean-all:
	make -C emsdk clean
	make -C cpython clean-all

src/core/error_handling_cpp.o: src/core/error_handling_cpp.cpp
	$(CXX) -o $@ -c $< $(MAIN_MODULE_CFLAGS) -Isrc/core/

%.o: %.c $(CPYTHONLIB) $(wildcard src/core/*.h src/core/*.js)
	$(CC) -o $@ -c $< $(MAIN_MODULE_CFLAGS) -Isrc/core/


# Stdlib modules that we repackage as standalone packages

# TODO: also include test directories included in other stdlib modules
build/test.tar: $(CPYTHONLIB) node_modules/.installed
	cd $(CPYTHONLIB) && tar --exclude=__pycache__ -cf $(PYODIDE_ROOT)/build/test.tar test

build/distutils.tar: $(CPYTHONLIB) node_modules/.installed
	cd $(CPYTHONLIB) && tar --exclude=__pycache__ -cf $(PYODIDE_ROOT)/build/distutils.tar distutils


$(CPYTHONLIB): emsdk/emsdk/.complete $(PYODIDE_EMCC) $(PYODIDE_CXX)
	date +"[%F %T] Building cpython..."
	make -C $(CPYTHONROOT)
	date +"[%F %T] done building cpython..."


build/packages.json: FORCE
	date +"[%F %T] Building packages..."
	make -C packages
	date +"[%F %T] done building packages..."


emsdk/emsdk/.complete:
	date +"[%F %T] Building emsdk..."
	make -C emsdk
	date +"[%F %T] done building emsdk."


FORCE:


check:
	./tools/dependency-check.sh


debug :
	EXTRA_CFLAGS+=" -D DEBUG_F" \
	make
