<commit_before># Makefile for Kapok
# -*- mode: Makefile -*-


MKDIR    := mkdir -p
RM       := rm -rf
CP       := cp -rf
REBAR    := rebar
ERL      := erl
ERLC     := erlc
ESCRIPT  := escript


QUIET    := @

lib_dir                := $(CURDIR)/lib
lib_names              := $(patsubst $(lib_dir)/%,%,$(wildcard $(lib_dir)/*))

# get erlang modules in specified directory
# $(call get-modules-in-dir,dir-name)
define get-modules-in-dir
  $(patsubst %.erl,%,$(notdir $(wildcard $1/*.erl)))
endef

# calculate beam file from module
# $(call modules-to-beams,destination,modules...)
define modules-to-beams
  $(addprefix $1/,$(addsuffix .beam,$2))
endef

# ensure directory's existence
# $(call ensure-dir,dir-name)
define ensure-dir
  $(QUIET) [ -d "$1" ] || $(MKDIR) $1
endef

# check the existence of command
# $(call check-cmd,cmd-name)
define check-cmd
  $(QUIET) if ! which "$1" > /dev/null; \
then echo "command" "$1" "not found"; exit 1; fi
endef

ERL_PATH_OPTIONS := -pa $(CURDIR)/*/*/ebin
ERL_OPTIONS      := -noshell $(ERL_PATH_OPTIONS)

# call erl command line
# $(call erl,file,function,arguments...)
define erl
  $(QUIET) $(ERL) $(ERL_OPTIONS) -s $1 $2 $3 -s init stop
endef

ERLC_OPTIONS      := $(ERL_PATH_OPTIONS)

# call erlc command line
# $(call erlc,file,outdir)
define erlc
  $(QUIET) $(ERLC) $(ERLC_OPTIONS) -o "$2" "$1"
endef

# call eunit test on specified module, using the auto-exported `test` function
# $(call eunit-test,module)
define eunit-test
  $(QUIET) $(ERL) $(ERL_OPTIONS) -s $1 test -s init stop
endef

# generate the targets with specified prefix for specified modules
# $(call gen-target,prefix,module...)
define gen-target
  $(addprefix $1,$2)
endef

# define the rules with specified prefix for the specified module
# $(call gen-run-test-rule,prefix,module)
define gen-run-test-rule
$1$2:
	$(QUIET) printf '--- Run test %s ---\n' $2
	$(call eunit-test,$2)
endef

# echo the message that about to build the specified type package
# $(call echo-build,type,module)
define echo-build
$(QUIET) printf "=== build %s %s ===\n" $1 $2
endef

# define the build vars for specified lib
# $(call gen-build-vars,lib)
define gen-build-vars

ifeq "$(strip $1)" "kapok"

$1_lib_dir             := $(lib_dir)/$1
$1_deps_dir            := $$($1_lib_dir)/deps
$1_src_dir             := $$($1_lib_dir)/src
$1_test_dir            := $$($1_lib_dir)/test
$1_beam_output_dir     := $$($1_lib_dir)/ebin
$1_test_output_dir     := $$($1_beam_output_dir)

$1_modules             := $$(call get-modules-in-dir,$$($1_src_dir))
$1_beam_files          := $$(call modules-to-beams,$$($1_beam_output_dir),$$($1_modules))
$1_test_modules        := $$(call get-modules-in-dir,$$($1_test_dir))
$1_test_beam_files     := $$(call modules-to-beams,$$($1_test_output_dir),$$($1_test_modules))
$1_parser_src_file     := $$($1_src_dir)/kapok_parser.erl
$1_other_files         := $$($1_lib_dir)/erl_crash.dump

else

$$(error "currently we don't define any vars for lib: $1")

endif

endef


# define the build rules for specified lib
# $(call gen-build-rules,lib,build-prefix,test-prefix,clean-prefix)
define gen-build-rules

ifeq "$(strip $1)" "kapok"

$2$1: $($1_parser_src_file) $($1_beam_files)

$($1_parser_src_file): $($1_src_dir)/%.erl:  $($1_src_dir)/%.yrl

$($1_parser_src_file):
	$(call echo-build,lib,$1)
	$(QUIET) echo "--- generate parser and build source files ---"
	$(QUIET) cd $($1_lib_dir) && $(REBAR) compile

$($1_beam_files): $($1_beam_output_dir)/%.beam: $($1_src_dir)/%.erl

$($1_beam_files):
	$(call echo-build,lib,$1)
	$(QUIET) echo "--- build source file ---"
	$(QUIET) cd $($1_lib_dir) && $(REBAR) compile

$3$1: build-test-$1 run-test-$1

build-test-$1: $($1_test_beam_files)

$($1_test_beam_files): $($1_beam_files)
$($1_test_output_dir)/%.beam: $($1_test_dir)/%.erl
	$$(call ensure-dir,$$(dir $$@))
	$$(QUIET) printf "Compile '%s' to beam\n" $$<
	$$(call erlc,$$<,$$(dir $$@))

run-test-$1: $($1_test_beam_files)
run-test-$1: $(call gen-target,run-test-,$($1_test_modules))
$(foreach m,$($1_test_modules), \
   $(eval $(call gen-run-test-rule,run-test-,$m)))

$4$1:
	$(RM) $($1_parser_src_file) $($1_beam_files) $($1_test_beam_files) $($1_other_files)

else

$$(error "currently we don't define any rule for lib: $1")

endif

endef

define gen-build-for
$(eval $(call gen-build-vars,$1))
$(eval $(call gen-build-rules,$1,$2,$3,$4))
endef
<commit_msg>add stdlib compilation<commit_after># Makefile for Kapok
# -*- mode: Makefile -*-


MKDIR    := mkdir -p
RM       := rm -rf
CP       := cp -rf
REBAR    := rebar
ERL      := erl
ERLC     := erlc
ESCRIPT  := escript
KAPOKC   := kapokc


QUIET    := @

lib_dir                := $(CURDIR)/lib
lib_names              := $(patsubst $(lib_dir)/%,%,$(wildcard $(lib_dir)/*))

# get files with specified suffix in specified directory
# $(call get-files-in-dir,dir-name,suffix)
define get-files-in-dir
  $(patsubst %.$2,%,$(notdir $(wildcard $1/*.$2)))
endef

# calculate beam file from module
# $(call modules-to-beams,destination,modules...)
define modules-to-beams
  $(addprefix $1/,$(addsuffix .beam,$2))
endef

# ensure directory's existence
# $(call ensure-dir,dir-name)
define ensure-dir
  $(QUIET) [ -d "$1" ] || $(MKDIR) $1
endef

# check the existence of command
# $(call check-cmd,cmd-name)
define check-cmd
  $(QUIET) if ! which "$1" > /dev/null; \
then echo "command" "$1" "not found"; exit 1; fi
endef

ERL_PATH_OPTIONS := -pa $(CURDIR)/*/*/ebin
ERL_OPTIONS      := -noshell $(ERL_PATH_OPTIONS)

# call erl command line
# $(call erl,file,function,arguments...)
define erl
  $(QUIET) $(ERL) $(ERL_OPTIONS) -s $1 $2 $3 -s init stop
endef

ERLC_OPTIONS      := $(ERL_PATH_OPTIONS)

# call erlc command line
# $(call erlc,file,outdir)
define erlc
  $(QUIET) $(ERLC) $(ERLC_OPTIONS) -o "$2" "$1"
endef

KAPOKC_OPTIONS    := $(ERLC_OPTIONS)

# call kapokc command line
# $(call kapokc,file,outdir)
define kapokc
  $(QUIET) $(KAPOKC) $(KAPOKC_OPTIONS) -o "$2" "$1"
endef

# call eunit test on specified module, using the auto-exported `test` function
# $(call eunit-test,module)
define eunit-test
  $(QUIET) $(ERL) $(ERL_OPTIONS) -s $1 test -s init stop
endef

# generate the targets with specified prefix for specified modules
# $(call gen-target,prefix,module...)
define gen-target
  $(addprefix $1,$2)
endef

# define the rules with specified prefix for the specified module
# $(call gen-run-test-rule,prefix,module)
define gen-run-test-rule
$1$2:
	$(QUIET) printf '--- Run test %s ---\n' $2
	$(call eunit-test,$2)
endef

# echo the message that about to build the specified type package
# $(call echo-build,type,module)
define echo-build
$(QUIET) printf "=== build %s %s ===\n" $1 $2
endef

# define the build vars for specified lib
# $(call gen-build-vars,lib)
define gen-build-vars

ifeq "$(strip $1)" "kapok"

lib_$1_dir             := $(lib_dir)/$1
$1_deps_dir            := $$(lib_$1_dir)/deps
$1_src_dir             := $$(lib_$1_dir)/src
$1_lib_dir             := $$(lib_$1_dir)/lib
$1_test_dir            := $$(lib_$1_dir)/test
$1_beam_output_dir     := $$(lib_$1_dir)/ebin

$1_modules             := $$(call get-files-in-dir,$$($1_src_dir),erl)
$1_lib_files           := $$(call get-files-in-dir,$$($1_lib_dir),kpk)
$1_beam_files          := $$(call modules-to-beams,$$($1_beam_output_dir),$$($1_modules))
$1_lib_beam_files      := $$(call modules-to-beams,$$($1_beam_output_dir),$$($1_lib_files))
$1_test_modules        := $$(call get-files-in-dir,$$($1_test_dir),erl)
$1_test_beam_files     := $$(call modules-to-beams,$$($1_beam_output_dir),$$($1_test_modules))
$1_parser_src_file     := $$($1_src_dir)/kapok_parser.erl
$1_other_files         := $$(lib_$1_dir)/erl_crash.dump

else

$$(error "currently we don't define any vars for lib: $1")

endif

endef


# define the build rules for specified lib
# $(call gen-build-rules,lib,build-prefix,test-prefix,clean-prefix)
define gen-build-rules

ifeq "$(strip $1)" "kapok"

$2$1: $2$1-compiler $2$1-libs

$2$1-compiler: $($1_parser_src_file) $($1_beam_files)

$($1_parser_src_file): $($1_src_dir)/%.erl:  $($1_src_dir)/%.yrl

$($1_parser_src_file):
	$(call echo-build,lib,$1)
	$(QUIET) echo "--- generate parser and build source files ---"
	$(QUIET) cd $(lib_$1_dir) && $(REBAR) compile

$($1_beam_files): $($1_beam_output_dir)/%.beam: $($1_src_dir)/%.erl

$($1_beam_files):
	$(call echo-build,lib,$1)
	$(QUIET) echo "--- build source file ---"
	$(QUIET) cd $($1_lib_dir) && $(REBAR) compile

$2$1-libs: $($1_lib_beam_files)

$($1_lib_beam_files): $($1_beam_output_dir)/%.beam: $($1_lib_dir)/%.kpk
	$(QUIET) printf "Compile '%s'\n" $$<
	$$(call kapokc,$$<,$$(dir $$@))

$3$1: build-test-$1 run-test-$1

build-test-$1: $2$1 $($1_test_beam_files)

$($1_test_beam_files): $($1_beam_files)
$($1_test_beam_files): $($1_beam_output_dir)/%.beam: $($1_test_dir)/%.erl
	$$(call ensure-dir,$$(dir $$@))
	$(QUIET) printf "Compile '%s'\n" $$<
	$$(call erlc,$$<,$$(dir $$@))

run-test-$1: $($1_test_beam_files)
run-test-$1: $(call gen-target,run-test-,$($1_test_modules))
$(foreach m,$($1_test_modules), \
   $(eval $(call gen-run-test-rule,run-test-,$m)))

$4$1:
	$(RM) $($1_parser_src_file) $($1_beam_files) $($1_test_beam_files) $($1_other_files)

else

$$(error "currently we don't define any rule for lib: $1")

endif

endef

define gen-build-for
$(eval $(call gen-build-vars,$1))
$(eval $(call gen-build-rules,$1,$2,$3,$4))
endef

{'fraction_non_alphanumeric': 0.1976918144902757, 'fraction_numerical': 0.022654413336182944, 'mean_word_length': 4.094719651605879, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7257379', 'n_tokens_mistral': 3846, 'n_tokens_neox': 3700, 'n_words': 829}