<commit_before>##
# This file should be included by extbld Makefiles.
#
# This also could generate md5sums for specified extbld Makefile:
# Assuming you are in ROOT_DIR, and you are going to generate md5sums of third_party/smth/, run
# make EXTBLD_LIB=$PWD/mk/extbld/lib.mk ROOT_DIR=$PWD -C third-party/smth BUILD_DIR=$PWD/build/extbld/third_party/smth md5_gen
#

ifeq ($(strip $(ROOT_DIR)),)
$(error ROOT_DIR is not set)
endif

ifeq ($(strip $(BUILD_DIR)),)
$(error BUILD_DIR is not set)
endif

include $(ROOT_DIR)/mk/core/common.mk
include $(ROOT_DIR)/mk/core/string.mk

.PHONY : all download extract patch configure build install
.PHONY : md5_gen

all : download extract patch configure build install

PKG_SOURCE_DIR   ?= $(BUILD_DIR)/$(PKG_NAME)-$(PKG_VER)
PKG_INSTALL_DIR  := $(BUILD_DIR)/install
DOWNLOAD_BASEDIR := $(ROOT_DIR)/download
ifeq ($(value PKG_DOWNLOADS_SEPARATE),)
DOWNLOAD_DIR     := $(DOWNLOAD_BASEDIR)
else
DOWNLOAD_DIR     := $(DOWNLOAD_BASEDIR)/$(PKG_NAME)
endif

$(DOWNLOAD_DIR) $(BUILD_DIR) $(PKG_INSTALL_DIR):
	mkdir -p $@

PKG_SOURCES ?=
sources_git      := $(filter %.git,$(PKG_SOURCES))
targets_git	  = $(basename $(notdir $1))
sources_download := $(filter-out %.git,$(PKG_SOURCES))
targets_download  = $(notdir $1)
sources_extract  := $(filter %.tar.gz %.tar.bz2 %tgz %tbz %zip,$(call targets_download,$(sources_download)))

DOWNLOAD  := $(BUILD_DIR)/.downloaded
$(DOWNLOAD): | $(DOWNLOAD_DIR) $(BUILD_DIR)
	$(foreach d,$(sources_download), \
		if [ ! -f $(DOWNLOAD_DIR)/$(call targets_download,$d) ]; then \
			cd $(DOWNLOAD_DIR) && (curl -O -k -L $d); \
		fi;)
	$(foreach g,$(sources_git), \
		if [ ! -d $(DOWNLOAD_DIR)/$(call targets_git,$g) ]; then \
			cd $(DOWNLOAD_DIR); \
			git clone $g; \
		fi;)
	touch $@

DOWNLOAD_CHECK  := $(BUILD_DIR)/.download_checked
$(DOWNLOAD_CHECK) : $(DOWNLOAD)
	$(if $(call eq,$(words $(PKG_SOURCES)),$(words $(PKG_MD5))),, \
		echo "different number of sources and MD5"; false)
	( cd $(DOWNLOAD_DIR); \
		$(foreach c,$(filter-out %.-,$(join $(PKG_SOURCES),$(addprefix .,$(PKG_MD5)))), \
			$(MD5) $(notdir $(basename $c)) | grep $(subst .,,$(suffix $c)) 2>&1 >/dev/null ; ) \
	)
	touch $@

download : $(DOWNLOAD) $(DOWNLOAD_CHECK)
md5_gen : $(DOWNLOAD)
	@echo PKG_MD5 := \\
	@$(foreach s,$(notdir $(PKG_SOURCES)),md5=$$(md5sum $(DOWNLOAD_DIR)/$s 2>/dev/null) && echo -e "\\t$${md5%%  *} \\" || echo -- "-";)
	@echo

EXTRACT  := $(BUILD_DIR)/.extracted
extract : $(EXTRACT)
$(EXTRACT): | $(DOWNLOAD_DIR) $(BUILD_DIR)
	$(foreach i,$(sources_extract),\
		$(if $(filter %zip,$i),unzip $(DOWNLOAD_DIR)/$i -d $(BUILD_DIR),\
			mkdir -p $(BUILD_DIR) && ( cd $(BUILD_DIR); tar -xf $(DOWNLOAD_DIR)/$i) );)
	COPY_FILES="$(addprefix $(DOWNLOAD_DIR)/, \
			$(call targets_git,$(sources_git)) \
			$(filter-out $(sources_extract),$(call targets_download,$(sources_download))))"; \
		if [ "$$COPY_FILES" ]; then \
			cp -R $$COPY_FILES $(BUILD_DIR); \
		fi
	touch $@

PATCH  := $(BUILD_DIR)/.patched
patch : $(PATCH)<commit_msg>mk: Add possibility to store third-party stuff with specified name<commit_after>##
# This file should be included by extbld Makefiles.
#
# This also could generate md5sums for specified extbld Makefile:
# Assuming you are in ROOT_DIR, and you are going to generate md5sums of third_party/smth/, run
# make EXTBLD_LIB=$PWD/mk/extbld/lib.mk ROOT_DIR=$PWD -C third-party/smth BUILD_DIR=$PWD/build/extbld/third_party/smth md5_gen
#

ifeq ($(strip $(ROOT_DIR)),)
$(error ROOT_DIR is not set)
endif

ifeq ($(strip $(BUILD_DIR)),)
$(error BUILD_DIR is not set)
endif

include $(ROOT_DIR)/mk/core/common.mk
include $(ROOT_DIR)/mk/core/string.mk

.PHONY : all download extract patch configure build install
.PHONY : md5_gen

all : download extract patch configure build install

PKG_SOURCE_DIR   ?= $(BUILD_DIR)/$(PKG_NAME)-$(PKG_VER)
PKG_INSTALL_DIR  := $(BUILD_DIR)/install
DOWNLOAD_BASEDIR := $(ROOT_DIR)/download
ifeq ($(value PKG_DOWNLOADS_SEPARATE),)
DOWNLOAD_DIR     := $(DOWNLOAD_BASEDIR)
else
DOWNLOAD_DIR     := $(DOWNLOAD_BASEDIR)/$(PKG_NAME)
endif

$(DOWNLOAD_DIR) $(BUILD_DIR) $(PKG_INSTALL_DIR):
	mkdir -p $@

PKG_SOURCES ?=
# Name of an archive where package will be saved to.
PKG_ARCHIVE_NAME ?=""
sources_git      := $(filter %.git,$(PKG_SOURCES))
targets_git	  = $(basename $(notdir $1))
sources_download := $(filter-out %.git,$(PKG_SOURCES))
# Return PKG_ARCHIVE_NAME of the specified package in case if
# the url does not contain package name
pkg_archive_name  = $(if $(call eq,$(PKG_ARCHIVE_NAME),""),$(call notdir,$1),$(PKG_ARCHIVE_NAME))
sources_extract  := $(filter %.tar.gz %.tar.bz2 %tgz %tbz %zip,$(call pkg_archive_name,$(sources_download)))

DOWNLOAD  := $(BUILD_DIR)/.downloaded
$(DOWNLOAD): | $(DOWNLOAD_DIR) $(BUILD_DIR)
	$(foreach d,$(sources_download), \
		if [ ! -f $(DOWNLOAD_DIR)/$(call pkg_archive_name,$d) ]; then \
			cd $(DOWNLOAD_DIR) && (curl -o $(call pkg_archive_name,$d) -k -L '$d'); \
		fi;)
	$(foreach g,$(sources_git), \
		if [ ! -d $(DOWNLOAD_DIR)/$(call targets_git,$g) ]; then \
			cd $(DOWNLOAD_DIR); \
			git clone $g; \
		fi;)
	touch $@

DOWNLOAD_CHECK  := $(BUILD_DIR)/.download_checked
$(DOWNLOAD_CHECK) : $(DOWNLOAD)
	$(if $(call eq,$(words $(PKG_SOURCES)),$(words $(PKG_MD5))),, \
		echo "different number of sources and MD5"; false)
	( cd $(DOWNLOAD_DIR); \
		$(foreach c,$(filter-out %.-,$(join $(PKG_SOURCES),$(addprefix .,$(PKG_MD5)))), \
			$(MD5) $(call pkg_archive_name,$(basename $c)) | grep $(subst .,,$(suffix $c)) 2>&1 >/dev/null ; ) \
	)
	touch $@

download : $(DOWNLOAD) $(DOWNLOAD_CHECK)
md5_gen : $(DOWNLOAD)
	@echo PKG_MD5 := \\
	@$(foreach s,$(PKG_SOURCES),md5=$$(md5sum $(DOWNLOAD_DIR)/$(call pkg_archive_name,$s) 2>/dev/null) && echo -e "\\t$${md5%%  *} \\" || echo -- "-";)
	@echo

EXTRACT  := $(BUILD_DIR)/.extracted
extract : $(EXTRACT)
$(EXTRACT): | $(DOWNLOAD_DIR) $(BUILD_DIR)
	$(foreach i,$(sources_extract),\
		$(if $(filter %zip,$i),unzip $(DOWNLOAD_DIR)/$i -d $(BUILD_DIR),\
			mkdir -p $(BUILD_DIR) && ( cd $(BUILD_DIR); tar -xf $(DOWNLOAD_DIR)/$i) );)
	COPY_FILES="$(addprefix $(DOWNLOAD_DIR)/, \
			$(call targets_git,$(sources_git)) \
			$(filter-out $(sources_extract),$(call pkg_archive_name,$(sources_download))))"; \
		if [ "$$COPY_FILES" ]; then \
			cp -R $$COPY_FILES $(BUILD_DIR); \
		fi
	touch $@

PATCH  := $(BUILD_DIR)/.patched
patch : $(PATCH)
{'fraction_non_alphanumeric': 0.19427305806043346, 'fraction_numerical': 0.006011707008384749, 'mean_word_length': 4.848288621646623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2046980', 'n_tokens_mistral': 2534, 'n_tokens_neox': 2401, 'n_words': 539}