<commit_before>#
# Build instructions
#


# Make sure that any failure in a pipe fails the build
SHELL = /bin/bash -o pipefail


# A list of all test names
TESTS ?= $(notdir $(basename $(wildcard test/*_test.nim)))


# Run all targets
.PHONY: all
all: readme test

# Run all tests
.PHONY: test
test: $(TESTS)


# Compiles a nim file
define COMPILE
nimble c $(FLAGS) \
		--path:. --nimcache:./build/nimcache --verbosity:0 \
		--out:../build/$(notdir $(basename $1)) \
		$1
endef


# A template for defining targets for a test
define DEFINE_TEST

.PHONY: build/$1
build/$1: test/$1.nim $(shell find -name $(patsubst %_test,%,$1).nim)

	$(call COMPILE,test/$1.nim)
	build/$1

.PHONY: $1
$1: build/$1

endef

# Define a target for each test
$(foreach test,$(TESTS),$(eval $(call DEFINE_TEST,$(test))))


# Pulls code snippets out of the readme file and puts them in their own files
define EXTRACT_README_CODE
import ropes, strutils
var blob = rope("")
var within = false
var count = 0
for line in lines("README.md"):
    if line.startsWith("```"):
        if within:
            writeFile("build/readme_" & $$count & ".nim", $$blob)
            inc(count)
            blob = rope("")
        within = not within
    elif within:
        blob.add(line)
        blob.add("\n")
endef
export EXTRACT_README_CODE

# A script that pulls code out of the readme. Source above
build/extract_readme_code: Makefile
	@mkdir -p build
	@echo "$$EXTRACT_README_CODE" > build/extract_readme_code.nim
	$(call COMPILE,build/extract_readme_code.nim)

# Execute the script to extract the source
build/readme_%.nim: README.md build/extract_readme_code
	@build/extract_readme_code
	@echo

# Compiles the code in the readme to make sure it works
build/readme_%: build/readme_%.nim
	@echo "Compiling $<"
	$(call COMPILE,$<)
	@echo
	$@
	@echo

# Define a general rule that compiles all the readme code
readme: $(addprefix build/readme_,$(shell seq 0 \
	$(shell expr $(shell cat README.md | grep '```' | wc -l) / 2 - 1)))


# Watches for changes and reruns
.PHONY: watch
watch:
	$(eval MAKEFLAGS += " -s ")
	@while true; do \
		make TESTS="$(TESTS)"; \
		inotifywait -qre close_write `find . -name "*.nim"` > /dev/null; \
		echo "Change detected, re-running..."; \
	done


# Executes the compiler with profiling enabled
.PHONY: profile
profile:
	make FLAGS="--profiler:on --stackTrace:on"


# Remove all build artifacts
.PHONY: clean
clean:
	rm -rf build

<commit_msg>Add README to watch<commit_after>#
# Build instructions
#


# Make sure that any failure in a pipe fails the build
SHELL = /bin/bash -o pipefail


# A list of all test names
TESTS ?= $(notdir $(basename $(wildcard test/*_test.nim)))


# Run all targets
.PHONY: all
all: readme test

# Run all tests
.PHONY: test
test: $(TESTS)


# Compiles a nim file
define COMPILE
nimble c $(FLAGS) \
		--path:. --nimcache:./build/nimcache --verbosity:0 \
		--out:../build/$(notdir $(basename $1)) \
		$1
endef


# A template for defining targets for a test
define DEFINE_TEST

.PHONY: build/$1
build/$1: test/$1.nim $(shell find -name $(patsubst %_test,%,$1).nim)

	$(call COMPILE,test/$1.nim)
	build/$1

.PHONY: $1
$1: build/$1

endef

# Define a target for each test
$(foreach test,$(TESTS),$(eval $(call DEFINE_TEST,$(test))))


# Pulls code snippets out of the readme file and puts them in their own files
define EXTRACT_README_CODE
import ropes, strutils
var blob = rope("")
var within = false
var count = 0
for line in lines("README.md"):
    if line.startsWith("```"):
        if within:
            writeFile("build/readme_" & $$count & ".nim", $$blob)
            inc(count)
            blob = rope("")
        within = not within
    elif within:
        blob.add(line)
        blob.add("\n")
endef
export EXTRACT_README_CODE

# A script that pulls code out of the readme. Source above
build/extract_readme_code: Makefile
	@mkdir -p build
	@echo "$$EXTRACT_README_CODE" > build/extract_readme_code.nim
	$(call COMPILE,build/extract_readme_code.nim)

# Execute the script to extract the source
build/readme_%.nim: README.md build/extract_readme_code
	@build/extract_readme_code
	@echo

# Compiles the code in the readme to make sure it works
build/readme_%: build/readme_%.nim
	@echo "Compiling $<"
	$(call COMPILE,$<)
	@echo
	$@
	@echo

# Define a general rule that compiles all the readme code
readme: $(addprefix build/readme_,$(shell seq 0 \
	$(shell expr $(shell cat README.md | grep '```' | wc -l) / 2 - 1)))


# Watches for changes and reruns
.PHONY: watch
watch:
	$(eval MAKEFLAGS += " -s ")
	@while true; do \
		make TESTS="$(TESTS)"; \
		inotifywait -qre close_write `find . -name "*.nim"` README.md \
			> /dev/null; \
		echo "Change detected, re-running..."; \
	done


# Executes the compiler with profiling enabled
.PHONY: profile
profile:
	make FLAGS="--profiler:on --stackTrace:on"


# Remove all build artifacts
.PHONY: clean
clean:
	rm -rf build


{'fraction_non_alphanumeric': 0.13641004696753115, 'fraction_numerical': 0.006534613028384725, 'mean_word_length': 3.4086408640864088, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1228317', 'n_tokens_mistral': 1804, 'n_tokens_neox': 1768, 'n_words': 562}