<commit_before># GNU Makefile

-include user.make

ifndef build
  build := release
endif

default: all

include Makerules

ifndef OUT
  OUT := build/$(build_prefix)$(build)$(build_suffix)
endif

include Makethird

# --- Configuration ---

# Do not specify CFLAGS or LIBS on the make invocation line - specify
# XCFLAGS or XLIBS instead. Make ignores any lines in the makefile that
# set a variable that was set on the command line.
CFLAGS += $(XCFLAGS) -Iinclude
LIBS += $(XLIBS) -lm

ifneq ($(threading),no)
  ifeq ($(HAVE_PTHREAD),yes)
	THREADING_CFLAGS := $(PTHREAD_CFLAGS) -DHAVE_PTHREAD
	THREADING_LIBS := $(PTHREAD_LIBS)
  endif
endif

ifeq ($(HAVE_WIN32),yes)
  WIN32_LIBS := -lcomdlg32 -lgdi32
  WIN32_LDFLAGS := -Wl,-subsystem,windows
endif

# --- Commands ---

ifneq ($(verbose),yes)
  QUIET_AR = @ echo "    AR $@" ;
  QUIET_RANLIB = @ echo "    RANLIB $@" ;
  QUIET_CC = @ echo "    CC $@" ;
  QUIET_CXX = @ echo "    CXX $@" ;
  QUIET_GEN = @ echo "    GEN $@" ;
  QUIET_LINK = @ echo "    LINK $@" ;
  QUIET_RM = @ echo "    RM $@" ;
  QUIET_TAGS = @ echo "    TAGS $@" ;
  QUIET_WINDRES = @ echo "    WINDRES $@" ;
  QUIET_OBJCOPY = @ echo "    OBJCOPY $@" ;
  QUIET_DLLTOOL = @ echo "    DLLTOOL $@" ;
  QUIET_GENDEF = @ echo "    GENDEF $@" ;
endif

MKTGTDIR = mkdir -p $(dir $@)
CC_CMD = $(QUIET_CC) $(MKTGTDIR) ; $(CC) $(CFLAGS) -MMD -MP -o $@ -c $<
CXX_CMD = $(QUIET_CXX) $(MKTGTDIR) ; $(CXX) $(CFLAGS) $(XCXXFLAGS) -MMD -MP -o $@ -c $<
AR_CMD = $(QUIET_AR) $(MKTGTDIR) ; $(AR) cr $@ $^
ifdef RANLIB
  RANLIB_CMD = $(QUIET_RANLIB) $(RANLIB) $@
endif
LINK_CMD = $(QUIET_LINK) $(MKTGTDIR) ; $(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
TAGS_CMD = $(QUIET_TAGS) ctags -R --c-kinds=+p --exclude=platform/python --exclude=platform/c++
WINDRES_CMD = $(QUIET_WINDRES) $(MKTGTDIR) ; $(WINDRES) $< $@
OBJCOPY_CMD = $(QUIET_OBJCOPY) $(MKTGTDIR) ; $(LD) -r -b binary -z noexecstack -o $@ $<
GENDEF_CMD = $(QUIET_GENDEF) gendef - $< > $@
DLLTOOL_CMD = $(QUIET_DLLTOOL) dlltool -d $< -D $(notdir $(^:%.def=%.dll)) -l $@

ifeq ($(shared),yes)
LINK_CMD = $(QUIET_LINK) $(MKTGTDIR) ; $(CC) $(LDFLAGS) -o $@ \
	$(filter-out %.$(SO),$^) \
	$(sort $(patsubst %,-L%,$(dir $(filter %.$(SO),$^)))) \
	$(patsubst lib%.$(SO),-l%,$(notdir $(filter %.$(SO),$^))) \
	$(LIBS)
endif

# --- Rules ---

$(OUT)/%.a :
	$(AR_CMD)
	$(RANLIB_CMD)

$(OUT)/%.exe: %.c
	$(LINK_CMD)

$(OUT)/%.$(SO):
	$(LINK_CMD) $(LIB_LDFLAGS) $(THIRD_LIBS) $(LIBCRYPTO_LIBS)

$(OUT)/%.def: $(OUT)/%.$(SO)
	$(GENDEF_CMD)

$(OUT)/%_$(SO).a: $(OUT)/%.def
	$(DLLTOOL_CMD)

$(OUT)/source/helpers/mu-threads/%.o : source/helpers/mu-threads/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THREADING_CFLAGS)

$(OUT)/source/helpers/pkcs7/%.o : source/helpers/pkcs7/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(LIBCRYPTO_CFLAGS)

$(OUT)/source/tools/%.o : source/tools/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(THIRD_CFLAGS) $(THREADING_CFLAGS)

$(OUT)/generated/%.o : generated/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) -O0

$(OUT)/platform/x11/%.o : platform/x11/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(X11_CFLAGS)

$(OUT)/platform/x11/curl/%.o : platform/x11/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(X11_CFLAGS) $(CURL_CFLAGS)

$(OUT)/platform/gl/%.o : platform/gl/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(THIRD_CFLAGS) $(THIRD_GLUT_CFLAGS)

ifeq ($(HAVE_OBJCOPY),yes)
  $(OUT)/source/fitz/noto.o : source/fitz/noto.c
	$(CC_CMD) $(WARNING_CFLAGS) -Wdeclaration-after-statement -DHAVE_OBJCOPY $(LIB_CFLAGS) $(THIRD_CFLAGS)
endif

$(OUT)/source/fitz/memento.o : source/fitz/memento.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THIRD_CFLAGS) -DMEMENTO_MUPDF_HACKS

$(OUT)/source/%.o : source/%.c
	$(CC_CMD) $(WARNING_CFLAGS) -Wdeclaration-after-statement $(LIB_CFLAGS) $(THIRD_CFLAGS)

$(OUT)/source/%.o : source/%.cpp
	$(CXX_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THIRD_CFLAGS)

ifeq ($(HAVE_TESSERACT),yes)
$(OUT)/source/fitz/tessocr.o : source/fitz/tessocr.cpp
	$(CXX_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THIRD_CFLAGS) $(TESSERACT_CFLAGS) $(TESSERACT_DEFINES) $(TESSERACT_LANGFLAGS)
endif

$(OUT)/platform/%.o : platform/%.c
	$(CC_CMD) $(WARNING_CFLAGS)

$(OUT)/%.o: %.rc
	$(WINDRES_CMD)

.PRECIOUS : $(OUT)/%.o # Keep intermediates from chained rules
.PRECIOUS : $(OUT)/%.exe # Keep intermediates from chained rules

# --- File lists ---

THIRD_OBJ := $(THIRD_SRC:%.c=$(OUT)/%.o)
THIRD_OBJ := $(THIRD_OBJ:%.cc=$(OUT)/%.o)
THIRD_OBJ := $(THIRD_OBJ:%.cpp=$(OUT)/%.o)

THIRD_GLUT_OBJ := $(THIRD_GLUT_SRC:%.c=$(OUT)/%.o)

MUPDF_SRC := $(sort $(wildcard source/fitz/*.c))
MUPDF_SRC += $(sort $(wildcard source/fitz/*.cpp))
MUPDF_SRC += $(sort $(wildcard source/pdf/*.c))
MUPDF_SRC += $(sort $(wildcard source/xps/*.c))
MUPDF_SRC += $(sort $(wildcard source/svg/*.c))
MUPDF_SRC += $(sort $(wildcard source/html/*.c))
MUPDF_SRC += $(sort $(wildcard source/reflow/*.c))
MUPDF_SRC += $(sort $(wildcard source/cbz/*.c))

MUPDF_OBJ := $(MUPDF_SRC:%.c=$(OUT)/%.o)
MUPDF_OBJ := $(MUPDF_OBJ:%.cpp=$(OUT)/%.o)

THREAD_SRC := source/helpers/mu-threads/mu-threads.c
THREAD_OBJ := $(THREAD_SRC:%.c=$(OUT)/%.o)

PKCS7_SRC += source/helpers/pkcs7/pkcs7-openssl.c
PKCS7_OBJ := $(PKCS7_SRC:%.c=$(OUT)/%.o)

# --- Generated embedded font files ---
# Note: The if-else chains here must match the equivalent tests in noto.c

HEXDUMP_SH := scripts/hexdump.sh

FONT_BIN := $(sort $(wildcard resources/fonts/urw/*.cff))

ifeq ($(filter -DTOFU_CJK,$(XCFLAGS)),)
  ifeq ($(filter -DTOFU_CJK_EXT,$(XCFLAGS)),)
    ifeq ($(filter -DTOFU_CJK_LANG,$(XCFLAGS)),)
      FONT_BIN += $(sort $(wildcard resources/fonts/han/*.ttc))
    else
      FONT_BIN += $(sort $(wildcard resources/fonts/droid/DroidSansFallbackFull.ttf))
    endif
  else
    FONT_BIN += $(sort $(wildcard resources/fonts/droid/DroidSansFallback.ttf))
  endif
endif

ifeq ($(filter -DTOFU,$(XCFLAGS)),)
  ifeq ($(filter -DTOFU_NOTO,$(XCFLAGS)),)
    FONT_BIN += $(sort $(wildcard resources/fonts/noto/*.otf))
    FONT_BIN += $(sort $(wildcard resources/fonts/noto/*.ttf))
  endif
  ifeq ($(filter -DTOFU_SIL,$(XCFLAGS)),)
    FONT_BIN += $(sort $(wildcard resources/fonts/sil/*.cff))
  endif
endif

FONT_GEN := $(FONT_BIN:%=generated/%.c)

generated/%.cff.c : %.cff $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<
generated/%.otf.c : %.otf $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<
generated/%.ttf.c : %.ttf $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<
generated/%.ttc.c : %.ttc $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<

ifeq ($(HAVE_OBJCOPY),yes)
  MUPDF_OBJ += $(FONT_BIN:%=$(OUT)/%.o)
  $(OUT)/%.cff.o : %.cff ; $(OBJCOPY_CMD)
  $(OUT)/%.otf.o : %.otf ; $(OBJCOPY_CMD)
  $(OUT)/%.ttf.o : %.ttf ; $(OBJCOPY_CMD)
  $(OUT)/%.ttc.o : %.ttc ; $(OBJCOPY_CMD)
else
  MUPDF_OBJ += $(FONT_GEN:%.c=$(OUT)/%.o)
endif

generate: $(FONT_GEN)

# --- Generated ICC profiles ---

source/fitz/icc/%.icc.h: resources/icc/%.icc<commit_msg>Makefile: Use filter-out to remove TOFU'd font files.<commit_after># GNU Makefile

-include user.make

ifndef build
  build := release
endif

default: all

include Makerules

ifndef OUT
  OUT := build/$(build_prefix)$(build)$(build_suffix)
endif

include Makethird

# --- Configuration ---

# Do not specify CFLAGS or LIBS on the make invocation line - specify
# XCFLAGS or XLIBS instead. Make ignores any lines in the makefile that
# set a variable that was set on the command line.
CFLAGS += $(XCFLAGS) -Iinclude
LIBS += $(XLIBS) -lm

ifneq ($(threading),no)
  ifeq ($(HAVE_PTHREAD),yes)
	THREADING_CFLAGS := $(PTHREAD_CFLAGS) -DHAVE_PTHREAD
	THREADING_LIBS := $(PTHREAD_LIBS)
  endif
endif

ifeq ($(HAVE_WIN32),yes)
  WIN32_LIBS := -lcomdlg32 -lgdi32
  WIN32_LDFLAGS := -Wl,-subsystem,windows
endif

# --- Commands ---

ifneq ($(verbose),yes)
  QUIET_AR = @ echo "    AR $@" ;
  QUIET_RANLIB = @ echo "    RANLIB $@" ;
  QUIET_CC = @ echo "    CC $@" ;
  QUIET_CXX = @ echo "    CXX $@" ;
  QUIET_GEN = @ echo "    GEN $@" ;
  QUIET_LINK = @ echo "    LINK $@" ;
  QUIET_RM = @ echo "    RM $@" ;
  QUIET_TAGS = @ echo "    TAGS $@" ;
  QUIET_WINDRES = @ echo "    WINDRES $@" ;
  QUIET_OBJCOPY = @ echo "    OBJCOPY $@" ;
  QUIET_DLLTOOL = @ echo "    DLLTOOL $@" ;
  QUIET_GENDEF = @ echo "    GENDEF $@" ;
endif

MKTGTDIR = mkdir -p $(dir $@)
CC_CMD = $(QUIET_CC) $(MKTGTDIR) ; $(CC) $(CFLAGS) -MMD -MP -o $@ -c $<
CXX_CMD = $(QUIET_CXX) $(MKTGTDIR) ; $(CXX) $(CFLAGS) $(XCXXFLAGS) -MMD -MP -o $@ -c $<
AR_CMD = $(QUIET_AR) $(MKTGTDIR) ; $(AR) cr $@ $^
ifdef RANLIB
  RANLIB_CMD = $(QUIET_RANLIB) $(RANLIB) $@
endif
LINK_CMD = $(QUIET_LINK) $(MKTGTDIR) ; $(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
TAGS_CMD = $(QUIET_TAGS) ctags -R --c-kinds=+p --exclude=platform/python --exclude=platform/c++
WINDRES_CMD = $(QUIET_WINDRES) $(MKTGTDIR) ; $(WINDRES) $< $@
OBJCOPY_CMD = $(QUIET_OBJCOPY) $(MKTGTDIR) ; $(LD) -r -b binary -z noexecstack -o $@ $<
GENDEF_CMD = $(QUIET_GENDEF) gendef - $< > $@
DLLTOOL_CMD = $(QUIET_DLLTOOL) dlltool -d $< -D $(notdir $(^:%.def=%.dll)) -l $@

ifeq ($(shared),yes)
LINK_CMD = $(QUIET_LINK) $(MKTGTDIR) ; $(CC) $(LDFLAGS) -o $@ \
	$(filter-out %.$(SO),$^) \
	$(sort $(patsubst %,-L%,$(dir $(filter %.$(SO),$^)))) \
	$(patsubst lib%.$(SO),-l%,$(notdir $(filter %.$(SO),$^))) \
	$(LIBS)
endif

# --- Rules ---

$(OUT)/%.a :
	$(AR_CMD)
	$(RANLIB_CMD)

$(OUT)/%.exe: %.c
	$(LINK_CMD)

$(OUT)/%.$(SO):
	$(LINK_CMD) $(LIB_LDFLAGS) $(THIRD_LIBS) $(LIBCRYPTO_LIBS)

$(OUT)/%.def: $(OUT)/%.$(SO)
	$(GENDEF_CMD)

$(OUT)/%_$(SO).a: $(OUT)/%.def
	$(DLLTOOL_CMD)

$(OUT)/source/helpers/mu-threads/%.o : source/helpers/mu-threads/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THREADING_CFLAGS)

$(OUT)/source/helpers/pkcs7/%.o : source/helpers/pkcs7/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(LIBCRYPTO_CFLAGS)

$(OUT)/source/tools/%.o : source/tools/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(THIRD_CFLAGS) $(THREADING_CFLAGS)

$(OUT)/generated/%.o : generated/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) -O0

$(OUT)/platform/x11/%.o : platform/x11/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(X11_CFLAGS)

$(OUT)/platform/x11/curl/%.o : platform/x11/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(X11_CFLAGS) $(CURL_CFLAGS)

$(OUT)/platform/gl/%.o : platform/gl/%.c
	$(CC_CMD) $(WARNING_CFLAGS) $(THIRD_CFLAGS) $(THIRD_GLUT_CFLAGS)

ifeq ($(HAVE_OBJCOPY),yes)
  $(OUT)/source/fitz/noto.o : source/fitz/noto.c
	$(CC_CMD) $(WARNING_CFLAGS) -Wdeclaration-after-statement -DHAVE_OBJCOPY $(LIB_CFLAGS) $(THIRD_CFLAGS)
endif

$(OUT)/source/fitz/memento.o : source/fitz/memento.c
	$(CC_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THIRD_CFLAGS) -DMEMENTO_MUPDF_HACKS

$(OUT)/source/%.o : source/%.c
	$(CC_CMD) $(WARNING_CFLAGS) -Wdeclaration-after-statement $(LIB_CFLAGS) $(THIRD_CFLAGS)

$(OUT)/source/%.o : source/%.cpp
	$(CXX_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THIRD_CFLAGS)

ifeq ($(HAVE_TESSERACT),yes)
$(OUT)/source/fitz/tessocr.o : source/fitz/tessocr.cpp
	$(CXX_CMD) $(WARNING_CFLAGS) $(LIB_CFLAGS) $(THIRD_CFLAGS) $(TESSERACT_CFLAGS) $(TESSERACT_DEFINES) $(TESSERACT_LANGFLAGS)
endif

$(OUT)/platform/%.o : platform/%.c
	$(CC_CMD) $(WARNING_CFLAGS)

$(OUT)/%.o: %.rc
	$(WINDRES_CMD)

.PRECIOUS : $(OUT)/%.o # Keep intermediates from chained rules
.PRECIOUS : $(OUT)/%.exe # Keep intermediates from chained rules

# --- File lists ---

THIRD_OBJ := $(THIRD_SRC:%.c=$(OUT)/%.o)
THIRD_OBJ := $(THIRD_OBJ:%.cc=$(OUT)/%.o)
THIRD_OBJ := $(THIRD_OBJ:%.cpp=$(OUT)/%.o)

THIRD_GLUT_OBJ := $(THIRD_GLUT_SRC:%.c=$(OUT)/%.o)

MUPDF_SRC := $(sort $(wildcard source/fitz/*.c))
MUPDF_SRC += $(sort $(wildcard source/fitz/*.cpp))
MUPDF_SRC += $(sort $(wildcard source/pdf/*.c))
MUPDF_SRC += $(sort $(wildcard source/xps/*.c))
MUPDF_SRC += $(sort $(wildcard source/svg/*.c))
MUPDF_SRC += $(sort $(wildcard source/html/*.c))
MUPDF_SRC += $(sort $(wildcard source/reflow/*.c))
MUPDF_SRC += $(sort $(wildcard source/cbz/*.c))

MUPDF_OBJ := $(MUPDF_SRC:%.c=$(OUT)/%.o)
MUPDF_OBJ := $(MUPDF_OBJ:%.cpp=$(OUT)/%.o)

THREAD_SRC := source/helpers/mu-threads/mu-threads.c
THREAD_OBJ := $(THREAD_SRC:%.c=$(OUT)/%.o)

PKCS7_SRC += source/helpers/pkcs7/pkcs7-openssl.c
PKCS7_OBJ := $(PKCS7_SRC:%.c=$(OUT)/%.o)

# --- Generated embedded font files ---

HEXDUMP_SH := scripts/hexdump.sh

FONT_BIN := $(sort $(wildcard resources/fonts/urw/*.cff))
FONT_BIN += $(sort $(wildcard resources/fonts/han/*.ttc))
FONT_BIN += $(sort $(wildcard resources/fonts/droid/DroidSansFallbackFull.ttf))
FONT_BIN += $(sort $(wildcard resources/fonts/droid/DroidSansFallback.ttf))
FONT_BIN += $(sort $(wildcard resources/fonts/noto/*.otf))
FONT_BIN += $(sort $(wildcard resources/fonts/noto/*.ttf))
FONT_BIN += $(sort $(wildcard resources/fonts/sil/*.cff))

# Note: The tests here must match the equivalent tests in noto.c

ifneq ($(filter -DTOFU_CJK,$(XCFLAGS)),)
  FONT_BIN := $(filter-out resources/fonts/han/%.ttc, $(FONT_BIN))
  FONT_BIN := $(filter-out resources/fonts/droid/DroidSansFallbackFull.ttf, $(FONT_BIN))
  FONT_BIN := $(filter-out resources/fonts/droid/DroidSansFallback.ttf, $(FONT_BIN))
endif

ifneq ($(filter -DTOFU_CJK_EXT,$(XCFLAGS)),)
  FONT_BIN := $(filter-out resources/fonts/han/%.ttc, $(FONT_BIN))
  FONT_BIN := $(filter-out resources/fonts/droid/DroidSansFallbackFull.ttf, $(FONT_BIN))
endif

ifneq ($(filter -DTOFU_CJK_LANG,$(XCFLAGS)),)
  FONT_BIN := $(filter-out resources/fonts/han/%.ttc, $(FONT_BIN))
endif

ifneq ($(filter -DTOFU,$(XCFLAGS)),)
  FONT_BIN := $(filter-out resources/fonts/noto/%.otf,$(FONT_BIN))
  FONT_BIN := $(filter-out resources/fonts/noto/%.ttf,$(FONT_BIN))
  FONT_BIN := $(filter-out resources/fonts/sil/%.cff,$(FONT_BIN))
endif

ifneq ($(filter -DTOFU_NOTO,$(XCFLAGS)),)
  FONT_BIN := $(filter-out resources/fonts/noto/%.otf,$(FONT_BIN))
  FONT_BIN := $(filter-out resources/fonts/noto/%.ttf,$(FONT_BIN))
endif

ifneq ($(filter -DTOFU_SIL,$(XCFLAGS)),)
  FONT_BIN := $(filter-out resources/fonts/sil/%.cff,$(FONT_BIN))
endif

FONT_GEN := $(FONT_BIN:%=generated/%.c)

generated/%.cff.c : %.cff $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<
generated/%.otf.c : %.otf $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<
generated/%.ttf.c : %.ttf $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<
generated/%.ttc.c : %.ttc $(HEXDUMP_SH) ; $(QUIET_GEN) $(MKTGTDIR) ; bash $(HEXDUMP_SH) > $@ $<

ifeq ($(HAVE_OBJCOPY),yes)
  MUPDF_OBJ += $(FONT_BIN:%=$(OUT)/%.o)
  $(OUT)/%.cff.o : %.cff ; $(OBJCOPY_CMD)
  $(OUT)/%.otf.o : %.otf ; $(OBJCOPY_CMD)
  $(OUT)/%.ttf.o : %.ttf ; $(OBJCOPY_CMD)
  $(OUT)/%.ttc.o : %.ttc ; $(OBJCOPY_CMD)
else
  MUPDF_OBJ += $(FONT_GEN:%.c=$(OUT)/%.o)
endif

generate: $(FONT_GEN)

# --- Generated ICC profiles ---

source/fitz/icc/%.icc.h: resources/icc/%.icc
{'fraction_non_alphanumeric': 0.22506149221098662, 'fraction_numerical': 0.004099480732440558, 'mean_word_length': 4.379272326350606, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2570150', 'n_tokens_mistral': 6668, 'n_tokens_neox': 6168, 'n_words': 1087}