<commit_before>#
#  Makefile for the Duktape development repo
#
#  This Makefile is intended for ONLY internal Duktape development
#  on Linux (or other UNIX-like operating systems), and covers:
#
#    - building the Duktape source distributable
#    - running test cases
#    - building the duktape.org website
#
#  The Makefile now also works in a very limited fashion with Cygwin,
#  you can 'make dist' as long as you have enough software installed.
#  The closure compiler requires a working JDK installation (JRE is
#  not enough, JAVA_HOME must point to the JDK), and a working Apache
#  'ant' command.
#
#  The source distributable has more platform neutral example Makefiles
#  for end user projects (though an end user should really just use their
#  own Makefile).
#
#  YOU SHOULD NOT COMPILE DUKTAPE WITH THIS MAKEFILE IN YOUR PROJECT!
#
#  Duktape command line tools are built by first creating a source dist
#  directory, and then using the sources from the dist directory for
#  compilation.  This is as close as possible to the sources used by an
#  end user, at the risk of accidentally polluting the dist directory.
#
#  When creating actual distributables, always clean first.
#
#  External projects are downloaded on-the-fly.  Clone git repos shallowly
#  (--depth 1) whenever possible.  With build-critical resources, use a
#  specific version instead of "trunk".
#
#  The makefile should work with -jN except for the 'clean' target, use:
#
#    $ make clean; make -j4
#

# A few commands which may need to be edited.  NodeJS is sometimes found
# as 'nodejs', sometimes as 'node'; sometimes 'node' is unrelated to NodeJS
# so check 'nodejs' first.
GIT:=$(shell which git)
NODE:=$(shell which nodejs node | head -1)
WGET:=$(shell which wget)
JAVA:=$(shell which java)
VALGRIND:=$(shell which valgrind)
PYTHON:=$(shell which python)

# Scrape version from the public header; convert from e.g. 10203 -> '1.2.3'
DUK_VERSION:=$(shell cat src/duk_api_public.h.in | grep define | grep DUK_VERSION | tr -s ' ' ' ' | cut -d ' ' -f 3 | tr -d 'L')
DUK_MAJOR:=$(shell echo "$(DUK_VERSION) / 10000" | bc)
DUK_MINOR:=$(shell echo "$(DUK_VERSION) % 10000 / 100" | bc)
DUK_PATCH:=$(shell echo "$(DUK_VERSION) % 100" | bc)
DUK_VERSION_FORMATTED:=$(DUK_MAJOR).$(DUK_MINOR).$(DUK_PATCH)
GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD)
GIT_DESCRIBE:=$(shell git describe --always --dirty)
ifeq ($(GIT_BRANCH),master)
GIT_INFO:=$(GIT_DESCRIBE)
else
GIT_INFO:=$(GIT_DESCRIBE)-$(GIT_BRANCH)
endif
BUILD_DATETIME:=$(shell date +%Y%m%d%H%M%S)

DISTSRCSEP = dist/src-separate
DISTSRCCOM = dist/src
DISTCMD = dist/examples/cmdline

DUKTAPE_SOURCES_COMBINED =	\
	$(DISTSRCCOM)/duktape.c

DUKTAPE_SOURCES_SEPARATE =	\
	$(DISTSRCSEP)/duk_util_hashbytes.c \
	$(DISTSRCSEP)/duk_util_hashprime.c \
	$(DISTSRCSEP)/duk_util_bitdecoder.c \
	$(DISTSRCSEP)/duk_util_bitencoder.c \
	$(DISTSRCSEP)/duk_util_tinyrandom.c \
	$(DISTSRCSEP)/duk_util_bufwriter.c \
	$(DISTSRCSEP)/duk_util_misc.c \
	$(DISTSRCSEP)/duk_alloc_default.c \
	$(DISTSRCSEP)/duk_debug_macros.c \
	$(DISTSRCSEP)/duk_debug_vsnprintf.c \
	$(DISTSRCSEP)/duk_debug_heap.c \
	$(DISTSRCSEP)/duk_debug_fixedbuffer.c \
	$(DISTSRCSEP)/duk_error_macros.c \
	$(DISTSRCSEP)/duk_error_longjmp.c \
	$(DISTSRCSEP)/duk_error_throw.c \
	$(DISTSRCSEP)/duk_error_augment.c \
	$(DISTSRCSEP)/duk_error_misc.c \
	$(DISTSRCSEP)/duk_heap_misc.c \
	$(DISTSRCSEP)/duk_heap_memory.c \
	$(DISTSRCSEP)/duk_heap_alloc.c \
	$(DISTSRCSEP)/duk_heap_refcount.c \
	$(DISTSRCSEP)/duk_heap_markandsweep.c \
	$(DISTSRCSEP)/duk_heap_hashstring.c \
	$(DISTSRCSEP)/duk_heap_stringtable.c \
	$(DISTSRCSEP)/duk_heap_stringcache.c \
	$(DISTSRCSEP)/duk_hstring_misc.c \
	$(DISTSRCSEP)/duk_hthread_misc.c \
	$(DISTSRCSEP)/duk_hthread_alloc.c \
	$(DISTSRCSEP)/duk_hthread_builtins.c \
	$(DISTSRCSEP)/duk_hthread_stacks.c \
	$(DISTSRCSEP)/duk_hobject_alloc.c \
	$(DISTSRCSEP)/duk_hobject_class.c \
	$(DISTSRCSEP)/duk_hobject_enum.c \
	$(DISTSRCSEP)/duk_hobject_props.c \
	$(DISTSRCSEP)/duk_hobject_finalizer.c \
	$(DISTSRCSEP)/duk_hobject_pc2line.c \
	$(DISTSRCSEP)/duk_hobject_misc.c \
	$(DISTSRCSEP)/duk_hbuffer_alloc.c \
	$(DISTSRCSEP)/duk_hbuffer_ops.c \
	$(DISTSRCSEP)/duk_hbufferobject_misc.c \
	$(DISTSRCSEP)/duk_unicode_tables.c \
	$(DISTSRCSEP)/duk_unicode_support.c \
	$(DISTSRCSEP)/duk_debugger.c \
	$(DISTSRCSEP)/duk_builtins.c \
	$(DISTSRCSEP)/duk_js_ops.c \
	$(DISTSRCSEP)/duk_js_var.c \
	$(DISTSRCSEP)/duk_numconv.c \
	$(DISTSRCSEP)/duk_api_stack.c \
	$(DISTSRCSEP)/duk_api_heap.c \
	$(DISTSRCSEP)/duk_api_call.c \
	$(DISTSRCSEP)/duk_api_compile.c \
	$(DISTSRCSEP)/duk_api_bytecode.c \
	$(DISTSRCSEP)/duk_api_codec.c \
	$(DISTSRCSEP)/duk_api_memory.c \
	$(DISTSRCSEP)/duk_api_string.c \
	$(DISTSRCSEP)/duk_api_object.c \
	$(DISTSRCSEP)/duk_api_buffer.c \
	$(DISTSRCSEP)/duk_api_var.c \
	$(DISTSRCSEP)/duk_api_logging.c \
	$(DISTSRCSEP)/duk_api_debug.c \
	$(DISTSRCSEP)/duk_lexer.c \
	$(DISTSRCSEP)/duk_tval.c \
	$(DISTSRCSEP)/duk_js_call.c \
	$(DISTSRCSEP)/duk_js_executor.c \
	$(DISTSRCSEP)/duk_js_compiler.c \
	$(DISTSRCSEP)/duk_regexp_compiler.c \
	$(DISTSRCSEP)/duk_regexp_executor.c \
	$(DISTSRCSEP)/duk_bi_duktape.c \
	$(DISTSRCSEP)/duk_bi_thread.c \
	$(DISTSRCSEP)/duk_bi_thrower.c \
	$(DISTSRCSEP)/duk_bi_array.c \
	$(DISTSRCSEP)/duk_bi_boolean.c \
	$(DISTSRCSEP)/duk_bi_date.c \
	$(DISTSRCSEP)/duk_bi_date_unix.c \
	$(DISTSRCSEP)/duk_bi_date_windows.c \
	$(DISTSRCSEP)/duk_bi_error.c \
	$(DISTSRCSEP)/duk_bi_function.c \
	$(DISTSRCSEP)/duk_bi_global.c \
	$(DISTSRCSEP)/duk_bi_json.c \
	$(DISTSRCSEP)/duk_bi_math.c \
	$(DISTSRCSEP)/duk_bi_number.c \
	$(DISTSRCSEP)/duk_bi_object.c \
	$(DISTSRCSEP)/duk_bi_regexp.c \
	$(DISTSRCSEP)/duk_bi_string.c \
	$(DISTSRCSEP)/duk_bi_proxy.c \
	$(DISTSRCSEP)/duk_bi_buffer.c \
	$(DISTSRCSEP)/duk_bi_pointer.c \
	$(DISTSRCSEP)/duk_bi_logger.c \
	$(DISTSRCSEP)/duk_selftest.c \
	$(DISTSRCSEP)/duk_strings.c

# Use combined sources for testing etc.
DUKTAPE_SOURCES = $(DUKTAPE_SOURCES_COMBINED)
#DUKTAPE_SOURCES = $(DUKTAPE_SOURCES_SEPARATE)

# Duktape command line tool - example of a main() program, used
# for unit testing
DUKTAPE_CMDLINE_SOURCES = \
	$(DISTCMD)/duk_cmdline.c \
	dist/examples/alloc-logging/duk_alloc_logging.c \
	dist/examples/alloc-torture/duk_alloc_torture.c \
	dist/examples/alloc-hybrid/duk_alloc_hybrid.c \
	dist/examples/debug-trans-socket/duk_trans_socket.c

# Compiler setup for Linux
CC	= gcc
GXX	= g++

CCOPTS_FEATURES =
#CCOPTS_FEATURES += -DDUK_OPT_NO_PACKED_TVAL
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_ALIGN=4
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_ALIGN=8
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_BYTEORDER=1      # little
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_BYTEORDER=2      # middle
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_BYTEORDER=3      # big
#CCOPTS_FEATURES += -DDUK_OPT_NO_REFERENCE_COUNTING
#CCOPTS_FEATURES += -DDUK_OPT_NO_MARK_AND_SWEEP
#CCOPTS_FEATURES += -DDUK_OPT_NO_VOLUNTARY_GC
CCOPTS_FEATURES += -DDUK_OPT_SEGFAULT_ON_PANIC       # segfault on panic allows valgrind to show stack trace on panic
CCOPTS_FEATURES += -DDUK_OPT_DPRINT_COLORS
CCOPTS_FEATURES += -DDUK_OPT_DPRINT_RDTSC
#CCOPTS_FEATURES += -DDUK_OPT_NO_FILE_IO
#CCOPTS_FEATURES += '-DDUK_OPT_PANIC_HANDLER(code,msg)={printf("*** %d:%s\n",(code),(msg));abort();}'
CCOPTS_FEATURES += -DDUK_OPT_SELF_TESTS
#CCOPTS_FEATURES += -DDUK_OPT_NO_TRACEBACKS
#CCOPTS_FEATURES += -DDUK_OPT_NO_PC2LINE
#CCOPTS_FEATURES += -DDUK_OPT_NO_VERBOSE_ERRORS
#CCOPTS_FEATURES += -DDUK_OPT_NO_VERBOSE_PROP_ERRORS
#CCOPTS_FEATURES += -DDUK_OPT_NO_AUGMENT_ERRORS
#CCOPTS_FEATURES += -DDUK_OPT_GC_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_SHUFFLE_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_REFZERO_FINALIZER_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_MARKANDSWEEP_FINALIZER_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_NO_MS_RESIZE_STRINGTABLE
CCOPTS_FEATURES += -DDUK_OPT_DEBUG_BUFSIZE=512
#CCOPTS_FEATURES += -DDUK_OPT_NO_STRICT_DECL<commit_msg>Makefile fixes for paranoid errors<commit_after>#
#  Makefile for the Duktape development repo
#
#  This Makefile is intended for ONLY internal Duktape development
#  on Linux (or other UNIX-like operating systems), and covers:
#
#    - building the Duktape source distributable
#    - running test cases
#    - building the duktape.org website
#
#  The Makefile now also works in a very limited fashion with Cygwin,
#  you can 'make dist' as long as you have enough software installed.
#  The closure compiler requires a working JDK installation (JRE is
#  not enough, JAVA_HOME must point to the JDK), and a working Apache
#  'ant' command.
#
#  The source distributable has more platform neutral example Makefiles
#  for end user projects (though an end user should really just use their
#  own Makefile).
#
#  YOU SHOULD NOT COMPILE DUKTAPE WITH THIS MAKEFILE IN YOUR PROJECT!
#
#  Duktape command line tools are built by first creating a source dist
#  directory, and then using the sources from the dist directory for
#  compilation.  This is as close as possible to the sources used by an
#  end user, at the risk of accidentally polluting the dist directory.
#
#  When creating actual distributables, always clean first.
#
#  External projects are downloaded on-the-fly.  Clone git repos shallowly
#  (--depth 1) whenever possible.  With build-critical resources, use a
#  specific version instead of "trunk".
#
#  The makefile should work with -jN except for the 'clean' target, use:
#
#    $ make clean; make -j4
#

# A few commands which may need to be edited.  NodeJS is sometimes found
# as 'nodejs', sometimes as 'node'; sometimes 'node' is unrelated to NodeJS
# so check 'nodejs' first.
GIT:=$(shell which git)
NODE:=$(shell which nodejs node | head -1)
WGET:=$(shell which wget)
JAVA:=$(shell which java)
VALGRIND:=$(shell which valgrind)
PYTHON:=$(shell which python)

# Scrape version from the public header; convert from e.g. 10203 -> '1.2.3'
DUK_VERSION:=$(shell cat src/duk_api_public.h.in | grep define | grep DUK_VERSION | tr -s ' ' ' ' | cut -d ' ' -f 3 | tr -d 'L')
DUK_MAJOR:=$(shell echo "$(DUK_VERSION) / 10000" | bc)
DUK_MINOR:=$(shell echo "$(DUK_VERSION) % 10000 / 100" | bc)
DUK_PATCH:=$(shell echo "$(DUK_VERSION) % 100" | bc)
DUK_VERSION_FORMATTED:=$(DUK_MAJOR).$(DUK_MINOR).$(DUK_PATCH)
GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD)
GIT_DESCRIBE:=$(shell git describe --always --dirty)
ifeq ($(GIT_BRANCH),master)
GIT_INFO:=$(GIT_DESCRIBE)
else
GIT_INFO:=$(GIT_DESCRIBE)-$(GIT_BRANCH)
endif
BUILD_DATETIME:=$(shell date +%Y%m%d%H%M%S)

DISTSRCSEP = dist/src-separate
DISTSRCCOM = dist/src
DISTCMD = dist/examples/cmdline

DUKTAPE_SOURCES_COMBINED =	\
	$(DISTSRCCOM)/duktape.c

DUKTAPE_SOURCES_SEPARATE =	\
	$(DISTSRCSEP)/duk_util_hashbytes.c \
	$(DISTSRCSEP)/duk_util_hashprime.c \
	$(DISTSRCSEP)/duk_util_bitdecoder.c \
	$(DISTSRCSEP)/duk_util_bitencoder.c \
	$(DISTSRCSEP)/duk_util_tinyrandom.c \
	$(DISTSRCSEP)/duk_util_bufwriter.c \
	$(DISTSRCSEP)/duk_util_misc.c \
	$(DISTSRCSEP)/duk_alloc_default.c \
	$(DISTSRCSEP)/duk_debug_macros.c \
	$(DISTSRCSEP)/duk_debug_vsnprintf.c \
	$(DISTSRCSEP)/duk_debug_heap.c \
	$(DISTSRCSEP)/duk_debug_fixedbuffer.c \
	$(DISTSRCSEP)/duk_error_macros.c \
	$(DISTSRCSEP)/duk_error_longjmp.c \
	$(DISTSRCSEP)/duk_error_throw.c \
	$(DISTSRCSEP)/duk_error_augment.c \
	$(DISTSRCSEP)/duk_error_misc.c \
	$(DISTSRCSEP)/duk_heap_misc.c \
	$(DISTSRCSEP)/duk_heap_memory.c \
	$(DISTSRCSEP)/duk_heap_alloc.c \
	$(DISTSRCSEP)/duk_heap_refcount.c \
	$(DISTSRCSEP)/duk_heap_markandsweep.c \
	$(DISTSRCSEP)/duk_heap_hashstring.c \
	$(DISTSRCSEP)/duk_heap_stringtable.c \
	$(DISTSRCSEP)/duk_heap_stringcache.c \
	$(DISTSRCSEP)/duk_hstring_misc.c \
	$(DISTSRCSEP)/duk_hthread_misc.c \
	$(DISTSRCSEP)/duk_hthread_alloc.c \
	$(DISTSRCSEP)/duk_hthread_builtins.c \
	$(DISTSRCSEP)/duk_hthread_stacks.c \
	$(DISTSRCSEP)/duk_hobject_alloc.c \
	$(DISTSRCSEP)/duk_hobject_class.c \
	$(DISTSRCSEP)/duk_hobject_enum.c \
	$(DISTSRCSEP)/duk_hobject_props.c \
	$(DISTSRCSEP)/duk_hobject_finalizer.c \
	$(DISTSRCSEP)/duk_hobject_pc2line.c \
	$(DISTSRCSEP)/duk_hobject_misc.c \
	$(DISTSRCSEP)/duk_hbuffer_alloc.c \
	$(DISTSRCSEP)/duk_hbuffer_ops.c \
	$(DISTSRCSEP)/duk_hbufferobject_misc.c \
	$(DISTSRCSEP)/duk_unicode_tables.c \
	$(DISTSRCSEP)/duk_unicode_support.c \
	$(DISTSRCSEP)/duk_debugger.c \
	$(DISTSRCSEP)/duk_builtins.c \
	$(DISTSRCSEP)/duk_js_ops.c \
	$(DISTSRCSEP)/duk_js_var.c \
	$(DISTSRCSEP)/duk_numconv.c \
	$(DISTSRCSEP)/duk_api_stack.c \
	$(DISTSRCSEP)/duk_api_heap.c \
	$(DISTSRCSEP)/duk_api_call.c \
	$(DISTSRCSEP)/duk_api_compile.c \
	$(DISTSRCSEP)/duk_api_bytecode.c \
	$(DISTSRCSEP)/duk_api_codec.c \
	$(DISTSRCSEP)/duk_api_memory.c \
	$(DISTSRCSEP)/duk_api_string.c \
	$(DISTSRCSEP)/duk_api_object.c \
	$(DISTSRCSEP)/duk_api_buffer.c \
	$(DISTSRCSEP)/duk_api_var.c \
	$(DISTSRCSEP)/duk_api_logging.c \
	$(DISTSRCSEP)/duk_api_debug.c \
	$(DISTSRCSEP)/duk_lexer.c \
	$(DISTSRCSEP)/duk_tval.c \
	$(DISTSRCSEP)/duk_js_call.c \
	$(DISTSRCSEP)/duk_js_executor.c \
	$(DISTSRCSEP)/duk_js_compiler.c \
	$(DISTSRCSEP)/duk_regexp_compiler.c \
	$(DISTSRCSEP)/duk_regexp_executor.c \
	$(DISTSRCSEP)/duk_bi_duktape.c \
	$(DISTSRCSEP)/duk_bi_thread.c \
	$(DISTSRCSEP)/duk_bi_thrower.c \
	$(DISTSRCSEP)/duk_bi_array.c \
	$(DISTSRCSEP)/duk_bi_boolean.c \
	$(DISTSRCSEP)/duk_bi_date.c \
	$(DISTSRCSEP)/duk_bi_date_unix.c \
	$(DISTSRCSEP)/duk_bi_date_windows.c \
	$(DISTSRCSEP)/duk_bi_error.c \
	$(DISTSRCSEP)/duk_bi_function.c \
	$(DISTSRCSEP)/duk_bi_global.c \
	$(DISTSRCSEP)/duk_bi_json.c \
	$(DISTSRCSEP)/duk_bi_math.c \
	$(DISTSRCSEP)/duk_bi_number.c \
	$(DISTSRCSEP)/duk_bi_object.c \
	$(DISTSRCSEP)/duk_bi_regexp.c \
	$(DISTSRCSEP)/duk_bi_string.c \
	$(DISTSRCSEP)/duk_bi_proxy.c \
	$(DISTSRCSEP)/duk_bi_buffer.c \
	$(DISTSRCSEP)/duk_bi_pointer.c \
	$(DISTSRCSEP)/duk_bi_logger.c \
	$(DISTSRCSEP)/duk_selftest.c \
	$(DISTSRCSEP)/duk_strings.c

# Use combined sources for testing etc.
DUKTAPE_SOURCES = $(DUKTAPE_SOURCES_COMBINED)
#DUKTAPE_SOURCES = $(DUKTAPE_SOURCES_SEPARATE)

# Duktape command line tool - example of a main() program, used
# for unit testing
DUKTAPE_CMDLINE_SOURCES = \
	$(DISTCMD)/duk_cmdline.c \
	dist/examples/alloc-logging/duk_alloc_logging.c \
	dist/examples/alloc-torture/duk_alloc_torture.c \
	dist/examples/alloc-hybrid/duk_alloc_hybrid.c \
	dist/examples/debug-trans-socket/duk_trans_socket.c

# Compiler setup for Linux
CC	= gcc
GXX	= g++

CCOPTS_FEATURES =
#CCOPTS_FEATURES += -DDUK_OPT_NO_PACKED_TVAL
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_ALIGN=4
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_ALIGN=8
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_BYTEORDER=1      # little
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_BYTEORDER=2      # middle
#CCOPTS_FEATURES += -DDUK_OPT_FORCE_BYTEORDER=3      # big
#CCOPTS_FEATURES += -DDUK_OPT_NO_REFERENCE_COUNTING
#CCOPTS_FEATURES += -DDUK_OPT_NO_MARK_AND_SWEEP
#CCOPTS_FEATURES += -DDUK_OPT_NO_VOLUNTARY_GC
CCOPTS_FEATURES += -DDUK_OPT_SEGFAULT_ON_PANIC       # segfault on panic allows valgrind to show stack trace on panic
CCOPTS_FEATURES += -DDUK_OPT_DPRINT_COLORS
CCOPTS_FEATURES += -DDUK_OPT_DPRINT_RDTSC
#CCOPTS_FEATURES += -DDUK_OPT_NO_FILE_IO
#CCOPTS_FEATURES += '-DDUK_OPT_PANIC_HANDLER(code,msg)={printf("*** %d:%s\n",(code),(msg));abort();}'
CCOPTS_FEATURES += -DDUK_OPT_SELF_TESTS
#CCOPTS_FEATURES += -DDUK_OPT_NO_TRACEBACKS
#CCOPTS_FEATURES += -DDUK_OPT_NO_PC2LINE
#CCOPTS_FEATURES += -DDUK_OPT_NO_VERBOSE_ERRORS
#CCOPTS_FEATURES += -DDUK_OPT_PARANOID_ERRORS
#CCOPTS_FEATURES += -DDUK_OPT_NO_AUGMENT_ERRORS
#CCOPTS_FEATURES += -DDUK_OPT_GC_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_SHUFFLE_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_REFZERO_FINALIZER_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_MARKANDSWEEP_FINALIZER_TORTURE
#CCOPTS_FEATURES += -DDUK_OPT_NO_MS_RESIZE_STRINGTABLE
CCOPTS_FEATURES += -DDUK_OPT_DEBUG_BUFSIZE=512
#CCOPTS_FEATURES += -DDUK_OPT_NO_STRICT_DECL
{'fraction_non_alphanumeric': 0.13041820261853312, 'fraction_numerical': 0.0048303038006864115, 'mean_word_length': 5.110679611650485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7495085', 'n_tokens_mistral': 7096, 'n_tokens_neox': 6771, 'n_words': 1065}