<commit_before>run : all

# ====== VERSIONS ======

ADVANCECOMP_VER := 1.19
GIFSICLE_VER := 1.86
JHEAD_VER := 2.97
JPEGOPTIM_VER := 1.4.1
LIBJPEG_VER := 9a
LIBPNG_VER := 1.6.13
LIBZ_VER := 1.2.8
OPTIPNG_VER := 0.7.5
PNGCRUSH_VER := 1.7.77
PNGQUANT_VER := 2.3.0

# ====== CONSTANTS ======

OS := $(shell uname -s | tr A-Z a-z)
ARCH := $(shell uname -m)

IS_DARWIN := $(findstring darwin,$(OS))
DLEXT := $(if $(IS_DARWIN),.dylib,.so)

DL_DIR := $(CURDIR)/download
BUILD_ROOT_DIR := $(CURDIR)/build
BUILD_DIR := $(BUILD_ROOT_DIR)/$(OS)/$(ARCH)
OUTPUT_ROOT_DIR := $(CURDIR)/vendor
OUTPUT_DIR := $(OUTPUT_ROOT_DIR)/$(OS)/$(ARCH)

# ====== HELPERS ======

downcase = $(shell echo $1 | tr A-Z a-z)

mkpath := mkdir -p
ln_s := ln -sf

# lock using %.lock dir, download to %.tmp rename to %, remove %.lock
define download
	while ! mkdir $2.lock 2> /dev/null; do sleep 1; done
	wget -q -O $2.tmp $1
	mv $2.tmp $2
	rm -r $2.lock
endef

# ====== ARCHIVES ======

$(shell $(mkpath) $(DL_DIR)) # just create dl dir

ARCHIVES :=

# $1 - name of archive
# $2 - url of archive with [VER] for replace with version
# $3 - optional addition to version string
define archive
$1_URL := $(subst [VER],$($1_VER)$(strip $3),$(strip $2))
$1_DIR := $(BUILD_DIR)/$(call downcase,$1)
$1_TGZ := $(DL_DIR)/$(call downcase,$1)-$($1_VER)$(strip $3).tar.gz
ARCHIVES += $1
# download archive from url
$$($1_TGZ) : ; $$(call download,$$($1_URL),$$@)
endef

$(eval $(call archive,ADVANCECOMP, http://prdownloads.sourceforge.net/advancemame/advancecomp-[VER].tar.gz?download))
$(eval $(call archive,GIFSICLE,    http://www.lcdf.org/gifsicle/gifsicle-[VER].tar.gz))
$(eval $(call archive,JHEAD,       http://www.sentex.net/~mwandel/jhead/jhead-[VER].tar.gz))
$(eval $(call archive,JPEGOPTIM,   http://www.kokkonen.net/tjko/src/jpegoptim-[VER].tar.gz))
$(eval $(call archive,LIBJPEG,     http://www.ijg.org/files/jpegsrc.v[VER].tar.gz))
$(eval $(call archive,LIBPNG,      http://prdownloads.sourceforge.net/libpng/libpng-[VER].tar.gz?download))
$(eval $(call archive,LIBZ,        http://prdownloads.sourceforge.net/libpng/zlib-[VER].tar.gz?download))
$(eval $(call archive,OPTIPNG,     http://prdownloads.sourceforge.net/optipng/optipng-[VER].tar.gz?download))
$(eval $(call archive,PNGCRUSH,    http://prdownloads.sourceforge.net/pmt/pngcrush-[VER].tar.gz?download))
$(eval $(call archive,PNGQUANT,    https://github.com/pornel/pngquant/archive/[VER].tar.gz))

# ====== PRODUCTS ======

$(shell $(mkpath) $(OUTPUT_DIR)) # just create output dir

PRODUCTS :=

# $1 - product name
# $2 - archive name ($1 if empty)
# $3 - basename ($1 if empty)
define target
$1_BASENAME := $(or $3,$(call downcase,$1))
$1_TARGET := $($(or $2,$1)_DIR)/$$($1_BASENAME)
PRODUCTS += $1
# first dependency on archive
$$($1_TARGET) $$($(or $2,$1)_DIR)/__$$(notdir $$($(or $2,$1)_TGZ))__ : $$($(or $2,$1)_TGZ)
# second dependency on check file
$$($1_TARGET) : $$($(or $2,$1)_DIR)/__$$(notdir $$($(or $2,$1)_TGZ))__
# copy product to output dir<commit_msg>add per product _DIR and _TGZ<commit_after>run : all

# ====== VERSIONS ======

ADVANCECOMP_VER := 1.19
GIFSICLE_VER := 1.86
JHEAD_VER := 2.97
JPEGOPTIM_VER := 1.4.1
LIBJPEG_VER := 9a
LIBPNG_VER := 1.6.13
LIBZ_VER := 1.2.8
OPTIPNG_VER := 0.7.5
PNGCRUSH_VER := 1.7.77
PNGQUANT_VER := 2.3.0

# ====== CONSTANTS ======

OS := $(shell uname -s | tr A-Z a-z)
ARCH := $(shell uname -m)

IS_DARWIN := $(findstring darwin,$(OS))
DLEXT := $(if $(IS_DARWIN),.dylib,.so)

DL_DIR := $(CURDIR)/download
BUILD_ROOT_DIR := $(CURDIR)/build
BUILD_DIR := $(BUILD_ROOT_DIR)/$(OS)/$(ARCH)
OUTPUT_ROOT_DIR := $(CURDIR)/vendor
OUTPUT_DIR := $(OUTPUT_ROOT_DIR)/$(OS)/$(ARCH)

# ====== HELPERS ======

downcase = $(shell echo $1 | tr A-Z a-z)

mkpath := mkdir -p
ln_s := ln -sf

# lock using %.lock dir, download to %.tmp rename to %, remove %.lock
define download
	while ! mkdir $2.lock 2> /dev/null; do sleep 1; done
	wget -q -O $2.tmp $1
	mv $2.tmp $2
	rm -r $2.lock
endef

# ====== ARCHIVES ======

$(shell $(mkpath) $(DL_DIR)) # just create dl dir

ARCHIVES :=

# $1 - name of archive
# $2 - url of archive with [VER] for replace with version
# $3 - optional addition to version string
define archive
$1_URL := $(subst [VER],$($1_VER)$(strip $3),$(strip $2))
$1_DIR := $(BUILD_DIR)/$(call downcase,$1)
$1_TGZ := $(DL_DIR)/$(call downcase,$1)-$($1_VER)$(strip $3).tar.gz
ARCHIVES += $1
# download archive from url
$$($1_TGZ) : ; $$(call download,$$($1_URL),$$@)
endef

$(eval $(call archive,ADVANCECOMP, http://prdownloads.sourceforge.net/advancemame/advancecomp-[VER].tar.gz?download))
$(eval $(call archive,GIFSICLE,    http://www.lcdf.org/gifsicle/gifsicle-[VER].tar.gz))
$(eval $(call archive,JHEAD,       http://www.sentex.net/~mwandel/jhead/jhead-[VER].tar.gz))
$(eval $(call archive,JPEGOPTIM,   http://www.kokkonen.net/tjko/src/jpegoptim-[VER].tar.gz))
$(eval $(call archive,LIBJPEG,     http://www.ijg.org/files/jpegsrc.v[VER].tar.gz))
$(eval $(call archive,LIBPNG,      http://prdownloads.sourceforge.net/libpng/libpng-[VER].tar.gz?download))
$(eval $(call archive,LIBZ,        http://prdownloads.sourceforge.net/libpng/zlib-[VER].tar.gz?download))
$(eval $(call archive,OPTIPNG,     http://prdownloads.sourceforge.net/optipng/optipng-[VER].tar.gz?download))
$(eval $(call archive,PNGCRUSH,    http://prdownloads.sourceforge.net/pmt/pngcrush-[VER].tar.gz?download))
$(eval $(call archive,PNGQUANT,    https://github.com/pornel/pngquant/archive/[VER].tar.gz))

# ====== PRODUCTS ======

$(shell $(mkpath) $(OUTPUT_DIR)) # just create output dir

PRODUCTS :=

# $1 - product name
# $2 - archive name ($1 if empty)
# $3 - basename ($1 if empty)
define target
$1_BASENAME := $(or $3,$(call downcase,$1))
$1_DIR := $($(or $2,$1)_DIR)
$1_TGZ := $($(or $2,$1)_TGZ)
$1_TARGET := $$($1_DIR)/$$($1_BASENAME)
PRODUCTS += $1
# first dependency on archive
$$($1_TARGET) $$($1_DIR)/__$$(notdir $$($1_TGZ))__ : $$($1_TGZ)
# second dependency on check file
$$($1_TARGET) : $$($1_DIR)/__$$(notdir $$($1_TGZ))__
# copy product to output dir
{'fraction_non_alphanumeric': 0.22438780609695153, 'fraction_numerical': 0.026653339996668333, 'mean_word_length': 4.569573283858998, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '839491', 'n_tokens_mistral': 2534, 'n_tokens_neox': 2409, 'n_words': 508}