<commit_before>include default.mk

GOOD_FEATURE_FILES = $(shell find testdata/good -name "*.feature")
BAD_FEATURE_FILES  = $(shell find testdata/bad -name "*.feature")

ASTS         = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.ast.ndjson,$(GOOD_FEATURE_FILES))
PICKLES      = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.pickles.ndjson,$(GOOD_FEATURE_FILES))
SOURCES      = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.source.ndjson,$(GOOD_FEATURE_FILES))
ERRORS       = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.errors.ndjson,$(BAD_FEATURE_FILES))

.DELETE_ON_ERROR:

default: .compared

.deps:
	./scripts/s3-download gherkin-go $(LIBRARY_VERSION)
	touch $@

.compared: $(ASTS) $(PICKLES) $(ERRORS) $(SOURCES)
	touch $@

# # Generate
# acceptance/testdata/%.feature.ast.ndjson: testdata/%.feature
# 	mkdir -p `dirname $@`
# 	bin/gherkin --no-source --no-pickles $< | jq --sort-keys --compact-output "." > $<.ast.ndjson

acceptance/testdata/%.feature.ast.ndjson: testdata/%.feature testdata/%.feature.ast.ndjson
	mkdir -p `dirname $@`
	bin/gherkin --no-source --no-pickles --json $< | jq --sort-keys --compact-output "." > $@
	diff --unified <(jq "." $<.ast.ndjson) <(jq "." $@)

# # Generate
# acceptance/testdata/%.feature.pickles.ndjson: testdata/%.feature
# 	mkdir -p `dirname $@`
# 	bin/gherkin --no-source --no-ast $< | jq --sort-keys --compact-output "." > $<.pickles.ndjson

acceptance/testdata/%.feature.pickles.ndjson: testdata/%.feature testdata/%.feature.pickles.ndjson
	mkdir -p `dirname $@`
	bin/gherkin --no-source --no-ast --json $< | jq --sort-keys --compact-output "." > $@
	diff --unified <(jq "." $<.pickles.ndjson) <(jq "." $@)

# # Generate
# acceptance/testdata/%.feature.source.ndjson: testdata/%.feature
# 	mkdir -p `dirname $@`
# 	bin/gherkin --no-ast --no-pickles $< | jq --sort-keys --compact-output "." > $<.source.ndjson

acceptance/testdata/%.feature.source.ndjson: testdata/%.feature testdata/%.feature.source.ndjson
	mkdir -p `dirname $@`
	bin/gherkin --no-ast --no-pickles --json $< | jq --sort-keys --compact-output "." > $@
	diff --unified <(jq "." $<.source.ndjson) <(jq "." $@)

# # Generate
# acceptance/testdata/%.feature.errors.ndjson: testdata/%.feature
# 	mkdir -p `dirname $@`
# 	bin/gherkin --no-source $< | jq --sort-keys --compact-output "." > $<.errors.ndjson

acceptance/testdata/%.feature.errors.ndjson: testdata/%.feature testdata/%.feature.errors.ndjson
	mkdir -p `dirname $@`
	bin/gherkin --no-source --json $< | jq --sort-keys --compact-output "." > $@
	diff --unified <(jq "." $<.errors.ndjson) <(jq "." $@)

clean:
	rm -rf .compared acceptance gherkin-go<commit_msg>gherkin: Disable tests for java<commit_after>include default.mk
#
#GOOD_FEATURE_FILES = $(shell find testdata/good -name "*.feature")
#BAD_FEATURE_FILES  = $(shell find testdata/bad -name "*.feature")
#
#ASTS         = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.ast.ndjson,$(GOOD_FEATURE_FILES))
#PICKLES      = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.pickles.ndjson,$(GOOD_FEATURE_FILES))
#SOURCES      = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.source.ndjson,$(GOOD_FEATURE_FILES))
#ERRORS       = $(patsubst testdata/%.feature,acceptance/testdata/%.feature.errors.ndjson,$(BAD_FEATURE_FILES))
#
#.DELETE_ON_ERROR:
#
#default: .compared
#
#.deps:
#	./scripts/s3-download gherkin-go $(LIBRARY_VERSION)
#	touch $@
#
#.compared: $(ASTS) $(PICKLES) $(ERRORS) $(SOURCES)
#	touch $@
#
## # Generate
## acceptance/testdata/%.feature.ast.ndjson: testdata/%.feature
## 	mkdir -p `dirname $@`
## 	bin/gherkin --no-source --no-pickles $< | jq --sort-keys --compact-output "." > $<.ast.ndjson
#
#acceptance/testdata/%.feature.ast.ndjson: testdata/%.feature testdata/%.feature.ast.ndjson
#	mkdir -p `dirname $@`
#	bin/gherkin --no-source --no-pickles --json $< | jq --sort-keys --compact-output "." > $@
#	diff --unified <(jq "." $<.ast.ndjson) <(jq "." $@)
#
## # Generate
## acceptance/testdata/%.feature.pickles.ndjson: testdata/%.feature
## 	mkdir -p `dirname $@`
## 	bin/gherkin --no-source --no-ast $< | jq --sort-keys --compact-output "." > $<.pickles.ndjson
#
#acceptance/testdata/%.feature.pickles.ndjson: testdata/%.feature testdata/%.feature.pickles.ndjson
#	mkdir -p `dirname $@`
#	bin/gherkin --no-source --no-ast --json $< | jq --sort-keys --compact-output "." > $@
#	diff --unified <(jq "." $<.pickles.ndjson) <(jq "." $@)
#
## # Generate
## acceptance/testdata/%.feature.source.ndjson: testdata/%.feature
## 	mkdir -p `dirname $@`
## 	bin/gherkin --no-ast --no-pickles $< | jq --sort-keys --compact-output "." > $<.source.ndjson
#
#acceptance/testdata/%.feature.source.ndjson: testdata/%.feature testdata/%.feature.source.ndjson
#	mkdir -p `dirname $@`
#	bin/gherkin --no-ast --no-pickles --json $< | jq --sort-keys --compact-output "." > $@
#	diff --unified <(jq "." $<.source.ndjson) <(jq "." $@)
#
## # Generate
## acceptance/testdata/%.feature.errors.ndjson: testdata/%.feature
## 	mkdir -p `dirname $@`
## 	bin/gherkin --no-source $< | jq --sort-keys --compact-output "." > $<.errors.ndjson
#
#acceptance/testdata/%.feature.errors.ndjson: testdata/%.feature testdata/%.feature.errors.ndjson
#	mkdir -p `dirname $@`
#	bin/gherkin --no-source --json $< | jq --sort-keys --compact-output "." > $@
#	diff --unified <(jq "." $<.errors.ndjson) <(jq "." $@)
#
#clean:
#	rm -rf .compared acceptance gherkin-go
{'fraction_non_alphanumeric': 0.21617161716171618, 'fraction_numerical': 0.0003667033370003667, 'mean_word_length': 3.618966977138019, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266549', 'n_tokens_mistral': 2171, 'n_tokens_neox': 2009, 'n_words': 339}