<commit_before>include ../VERSION.tmp

include ../iRODS/config/config.mk

include ../iRODS/config/external_versions.txt

MAKEFLAGS += --no-print-directory

RENCI_FTP_EXTERNAL=ftp://ftp.renci.org/pub/irods/external
RENCI_FTP_BUILD=ftp://ftp.renci.org/pub/irods/build

.PHONY : default build_database_for_macosx retrieve build generate clean cmake check libarchive jansson boost hdf5 netcdf epm lib_mysqludf_preg libs3

OSNAME=$$(../packaging/find_os.sh)
OSVERSION=$$(../packaging/find_os_version.sh)
OS=$(OSNAME)$$( if [ "$(OSNAME)" = "RedHatCompatible" -a "$(OSVERSION)" \> "6" ] ; then echo "_CentOS6" ; fi)
ifeq ($(OS_platform), osx_platform)
OS=$(OSNAME)$$( echo "-$(OSVERSION)" )
endif
IRODSEXTERNALARCHIVE=irods_external_$(IRODSVERSION)_$(OS).tar.gz
IRODSEXTERNALLOCAL=irods_external.tar.gz

CMAKEVERSIONARCHIVE=$(CMAKEVERSION).tar.gz
CMAKEVERSIONMAKE=$(CMAKEVERSION)/Makefile
CHECKVERSIONARCHIVE=$(CHECKVERSION).tar.gz
CHECKVERSIONMAKE=$(CHECKVERSION)/Makefile
LIBARCHIVEVERSIONARCHIVE=$(LIBARCHIVEVERSION).tar.gz
LIBARCHIVEVERSIONMAKE=$(LIBARCHIVEVERSION)/Makefile
JANSSONVERSIONARCHIVE=$(JANSSONVERSION).tar.gz
JANSSONVERSIONMAKE=$(JANSSONVERSION)/Makefile
BOOSTVERSIONARCHIVE=$(BOOSTVERSION).tar.gz
BOOSTVERSIONMAKE=$(BOOSTVERSION)/b2
HDF5VERSIONARCHIVE=$(HDF5VERSION).tar.gz
HDF5VERSIONMAKE=$(HDF5VERSION)/Makefile
NETCDFVERSIONARCHIVE=$(NETCDFVERSION).tar.gz
NETCDFVERSIONMAKE=$(NETCDFVERSION)/Makefile
EPMVERSIONARCHIVE=$(EPMVERSION).tar.gz
EPMVERSIONMAKE=epm/Makefile
LIB_MYSQLUDF_PREG_ARCHIVE=$(LIB_MYSQLUDF_PREG).tar.gz
LIB_MYSQLUDF_PREG_MAKE=lib_mysqludf_preg/Makefile
LIBS3VERSIONARCHIVE=$(LIBS3VERSION).tar.gz
LIBS3VERSIONMAKE=$(LIBS3VERSION)/GNUmakefile
AVROVERSIONARCHIVE=$(AVROVERSION).tar.gz
AVROVERSIONMAKE=$(AVROVERSION)/Makefile
ZEROMQVERSIONARCHIVE=$(ZEROMQVERSION).tar.gz
ZEROMQVERSIONCONFIGURE=$(ZEROMQVERSION)/configure
ZEROMQVERSIONMAKE=$(ZEROMQVERSION)/Makefile
CPPZMQVERSIONARCHIVE=$(CPPZMQVERSION).tar.gz
CPPZMQVERSIONMAKE=$(CPPZMQVERSION)/Makefile

FAKETARGETFORMAKE=FakeTargetForMake

LIBARCHIVESEDCMD="/^\#ifdef HAVE_LINUX_FIEMAP_H/i \#ifdef HAVE_LINUX_TYPES_H\x0a\#include \<linux/types.h\>\x0a\#endif"


PG_VERSION=9.3.4
UNIXODBC_VERSION=2.2.12
PGUNIXODBCARCHIVE=postgresql-$(PG_VERSION)_unixODBC-$(UNIXODBC_VERSION)-$(OSNAME).tar.gz

ifeq ($(OS_platform), osx_platform)
default : generate
else
default :
	@echo "Getting $(IRODSEXTERNALARCHIVE) -> $(IRODSEXTERNALLOCAL)..."
	$(V_at)curl -o $(IRODSEXTERNALLOCAL) -z $(IRODSEXTERNALLOCAL) $(RENCI_FTP_BUILD)/$(IRODSEXTERNALARCHIVE) > /dev/null 2>&1
	$(V_at)$(MAKE) $(FAKETARGETFORMAKE)
endif

$(PGUNIXODBCARCHIVE) :
	echo "Getting PostgreSQL and unixODBC source for MacOSX..."
	$(V_at)curl -o $(PGUNIXODBCARCHIVE) -z $(PGUNIXODBCARCHIVE) $(RENCI_FTP_EXTERNAL)/$(PGUNIXODBCARCHIVE) > /dev/null 2>&1;

postgresql-$(PG_VERSION)/Makefile : $(PGUNIXODBCARCHIVE)
	tar zxf $(PGUNIXODBCARCHIVE); \
		cd postgresql-$(PG_VERSION); \
		./configure --prefix=`pwd`/pgsql --without-readline --without-zlib;

build_database_for_macosx : postgresql-$(PG_VERSION)/Makefile
	cd postgresql-$(PG_VERSION); \
		$(MAKE); \
		$(MAKE) install; \
		cd src/interfaces/odbc/unixODBC-$(UNIXODBC_VERSION); \
		./configure --prefix=`pwd`/../../../../pgsql --enable-gui=no --enable-static; \
		$(MAKE); \
		$(MAKE) install; \
		$(MAKE) distclean; \
		cd ../../../../; \
		$(MAKE) distclean;

$(FAKETARGETFORMAKE) : $(IRODSEXTERNALLOCAL)
	@echo "Unpacking $(IRODSEXTERNALLOCAL)..."
	$(V_at)tar zxf $(IRODSEXTERNALLOCAL)
	$(V_at)touch $(FAKETARGETFORMAKE)

ifeq ($(OS_platform), osx_platform)
build : cmake libarchive jansson boost epm hdf5 netcdf lib_mysqludf_preg libs3 check avro zeromq build_database_for_macosx
else
build : cmake libarchive jansson boost epm hdf5 netcdf lib_mysqludf_preg libs3 check avro zeromq
endif

generate : build
	@echo "Building $(IRODSEXTERNALARCHIVE)..."
	$(V_at)tar zfc $(IRODSEXTERNALARCHIVE) --exclude "*.tar.gz" --exclude "*.zip" --exclude "*cmake*" --exclude "Makefile" --exclude $(FAKETARGETFORMAKE) *
	$(V_at)cp $(IRODSEXTERNALARCHIVE) $(IRODSEXTERNALLOCAL)
	$(V_at)touch $(FAKETARGETFORMAKE)

$(LIB_MYSQLUDF_PREG_ARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(LIB_MYSQLUDF_PREG_ARCHIVE) 2>&1

$(LIB_MYSQLUDF_PREG_MAKE) : $(LIB_MYSQLUDF_PREG_ARCHIVE)
	@tar xf $(LIB_MYSQLUDF_PREG_ARCHIVE)
	@+cd lib_mysqludf_preg ; ./configure

$(LIB_MYSQLUDF_PREG) : $(LIB_MYSQLUDF_PREG_MAKE)
	@$(MAKE) -C $(LIB_MYSQLUDF_PREG)

$(LIBS3VERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(LIBS3VERSIONARCHIVE) 2>&1

$(LIBS3VERSIONMAKE) : $(LIBS3VERSIONARCHIVE)
	@tar zxf $(LIBS3VERSIONARCHIVE)

libs3 : $(LIBS3VERSIONMAKE)
	$(V_at)if [ $(OSNAME) = "MacOSX" ] ; then \
		CFLAGS=-fPIC $(MAKE) -C $(LIBS3VERSION) -f GNUmakefile.osx; \
		else \
		CFLAGS=-fPIC $(MAKE) -C $(LIBS3VERSION); \
		fi

$(CMAKEVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(CMAKEVERSIONARCHIVE) 2>&1

$(CMAKEVERSIONMAKE) : $(CMAKEVERSIONARCHIVE)
	@tar xf $(CMAKEVERSIONARCHIVE) # confusingly, cmake archive is not actually gzipped, just tarred
	@sed "s/std::cerr << \"loading initial cache file \"/std::cout << \"loading initial cache file \"/" $(CMAKEVERSION)/Source/cmake.cxx > /tmp/cmake.cxx
	@rsync -c /tmp/cmake.cxx $(CMAKEVERSION)/Source/cmake.cxx
	@rm -f /tmp/cmake.cxx
	@+cd $(CMAKEVERSION) ; \
		./bootstrap -- -DBUILD_TESTING=FALSE

cmake : $(CMAKEVERSIONMAKE)
	@$(MAKE) -C $(CMAKEVERSION)

$(CHECKVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(CHECKVERSIONARCHIVE) 2>&1

$(CHECKVERSIONMAKE) : $(CHECKVERSIONARCHIVE)
	@tar zxf $(CHECKVERSIONARCHIVE)
	@cd $(CHECKVERSION) ; \
		./configure --with-pic

check : $(CHECKVERSIONMAKE)
	@$(MAKE) -C $(CHECKVERSION)

$(JANSSONVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(JANSSONVERSIONARCHIVE) 2>&1

$(JANSSONVERSIONMAKE) : $(JANSSONVERSIONARCHIVE)
	@tar zxf $(JANSSONVERSIONARCHIVE)
	@cd $(JANSSONVERSION) ; \
		./configure --with-pic

jansson : $(JANSSONVERSIONMAKE)
	@$(MAKE) -C $(JANSSONVERSION)

$(LIBARCHIVEVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(LIBARCHIVEVERSIONARCHIVE) 2>&1

$(LIBARCHIVEVERSIONMAKE) : $(LIBARCHIVEVERSIONARCHIVE) cmake
	@tar zxf $(LIBARCHIVEVERSIONARCHIVE)
	@if [ $(OSNAME) != "MacOSX" ] ; then \
		bash -xc 'sed $(LIBARCHIVESEDCMD) $(LIBARCHIVEVERSION)/libarchive/test/test_sparse_basic.c > /tmp/test_sparse_basic.c' ; \
		rsync -c /tmp/test_sparse_basic.c $(LIBARCHIVEVERSION)/libarchive/test/test_sparse_basic.c ; \
		rm -f /tmp/test_sparse_basic.c ; \
		fi
	@sed "s/size_t used\;/size_t used = 0\;/" $(LIBARCHIVEVERSION)/libarchive/archive_read_support_format_xar.c > /tmp/archive_read_support_format_xar.c
	@rsync -c /tmp/archive_read_support_format_xar.c $(LIBARCHIVEVERSION)/libarchive/archive_read_support_format_xar.c
	@rm -f /tmp/archive_read_support_format_xar.c
	@cd $(LIBARCHIVEVERSION) ; \
		../$(CMAKEVERSION)/bin/cmake -D CMAKE_C_FLAGS:STRING=-fPIC .

libarchive : $(LIBARCHIVEVERSIONMAKE)
	@$(MAKE) -C $(LIBARCHIVEVERSION)

$(BOOSTVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(BOOSTVERSIONARCHIVE) 2>&1

$(BOOSTVERSIONMAKE) : $(BOOSTVERSIONARCHIVE)
	@tar zxf $(BOOSTVERSIONARCHIVE)
	@cd $(BOOSTVERSION) ; \
		./bootstrap.sh --with-libraries=program_options,filesystem,system,thread,regex,program_options,iostreams,chrono,date_time
<commit_msg>[#2709] remove libarchive cachefile if exists<commit_after>include ../VERSION.tmp

include ../iRODS/config/config.mk

include ../iRODS/config/external_versions.txt

MAKEFLAGS += --no-print-directory

RENCI_FTP_EXTERNAL=ftp://ftp.renci.org/pub/irods/external
RENCI_FTP_BUILD=ftp://ftp.renci.org/pub/irods/build

.PHONY : default build_database_for_macosx retrieve build generate clean cmake check libarchive jansson boost hdf5 netcdf epm lib_mysqludf_preg libs3

OSNAME=$$(../packaging/find_os.sh)
OSVERSION=$$(../packaging/find_os_version.sh)
OS=$(OSNAME)$$( if [ "$(OSNAME)" = "RedHatCompatible" -a "$(OSVERSION)" \> "6" ] ; then echo "_CentOS6" ; fi)
ifeq ($(OS_platform), osx_platform)
OS=$(OSNAME)$$( echo "-$(OSVERSION)" )
endif
IRODSEXTERNALARCHIVE=irods_external_$(IRODSVERSION)_$(OS).tar.gz
IRODSEXTERNALLOCAL=irods_external.tar.gz

CMAKEVERSIONARCHIVE=$(CMAKEVERSION).tar.gz
CMAKEVERSIONMAKE=$(CMAKEVERSION)/Makefile
CHECKVERSIONARCHIVE=$(CHECKVERSION).tar.gz
CHECKVERSIONMAKE=$(CHECKVERSION)/Makefile
LIBARCHIVEVERSIONARCHIVE=$(LIBARCHIVEVERSION).tar.gz
LIBARCHIVEVERSIONMAKE=$(LIBARCHIVEVERSION)/Makefile
JANSSONVERSIONARCHIVE=$(JANSSONVERSION).tar.gz
JANSSONVERSIONMAKE=$(JANSSONVERSION)/Makefile
BOOSTVERSIONARCHIVE=$(BOOSTVERSION).tar.gz
BOOSTVERSIONMAKE=$(BOOSTVERSION)/b2
HDF5VERSIONARCHIVE=$(HDF5VERSION).tar.gz
HDF5VERSIONMAKE=$(HDF5VERSION)/Makefile
NETCDFVERSIONARCHIVE=$(NETCDFVERSION).tar.gz
NETCDFVERSIONMAKE=$(NETCDFVERSION)/Makefile
EPMVERSIONARCHIVE=$(EPMVERSION).tar.gz
EPMVERSIONMAKE=epm/Makefile
LIB_MYSQLUDF_PREG_ARCHIVE=$(LIB_MYSQLUDF_PREG).tar.gz
LIB_MYSQLUDF_PREG_MAKE=lib_mysqludf_preg/Makefile
LIBS3VERSIONARCHIVE=$(LIBS3VERSION).tar.gz
LIBS3VERSIONMAKE=$(LIBS3VERSION)/GNUmakefile
AVROVERSIONARCHIVE=$(AVROVERSION).tar.gz
AVROVERSIONMAKE=$(AVROVERSION)/Makefile
ZEROMQVERSIONARCHIVE=$(ZEROMQVERSION).tar.gz
ZEROMQVERSIONCONFIGURE=$(ZEROMQVERSION)/configure
ZEROMQVERSIONMAKE=$(ZEROMQVERSION)/Makefile
CPPZMQVERSIONARCHIVE=$(CPPZMQVERSION).tar.gz
CPPZMQVERSIONMAKE=$(CPPZMQVERSION)/Makefile

FAKETARGETFORMAKE=FakeTargetForMake

LIBARCHIVESEDCMD="/^\#ifdef HAVE_LINUX_FIEMAP_H/i \#ifdef HAVE_LINUX_TYPES_H\x0a\#include \<linux/types.h\>\x0a\#endif"


PG_VERSION=9.3.4
UNIXODBC_VERSION=2.2.12
PGUNIXODBCARCHIVE=postgresql-$(PG_VERSION)_unixODBC-$(UNIXODBC_VERSION)-$(OSNAME).tar.gz

ifeq ($(OS_platform), osx_platform)
default : generate
else
default :
	@echo "Getting $(IRODSEXTERNALARCHIVE) -> $(IRODSEXTERNALLOCAL)..."
	$(V_at)curl -o $(IRODSEXTERNALLOCAL) -z $(IRODSEXTERNALLOCAL) $(RENCI_FTP_BUILD)/$(IRODSEXTERNALARCHIVE) > /dev/null 2>&1
	$(V_at)$(MAKE) $(FAKETARGETFORMAKE)
endif

$(PGUNIXODBCARCHIVE) :
	echo "Getting PostgreSQL and unixODBC source for MacOSX..."
	$(V_at)curl -o $(PGUNIXODBCARCHIVE) -z $(PGUNIXODBCARCHIVE) $(RENCI_FTP_EXTERNAL)/$(PGUNIXODBCARCHIVE) > /dev/null 2>&1;

postgresql-$(PG_VERSION)/Makefile : $(PGUNIXODBCARCHIVE)
	tar zxf $(PGUNIXODBCARCHIVE); \
		cd postgresql-$(PG_VERSION); \
		./configure --prefix=`pwd`/pgsql --without-readline --without-zlib;

build_database_for_macosx : postgresql-$(PG_VERSION)/Makefile
	cd postgresql-$(PG_VERSION); \
		$(MAKE); \
		$(MAKE) install; \
		cd src/interfaces/odbc/unixODBC-$(UNIXODBC_VERSION); \
		./configure --prefix=`pwd`/../../../../pgsql --enable-gui=no --enable-static; \
		$(MAKE); \
		$(MAKE) install; \
		$(MAKE) distclean; \
		cd ../../../../; \
		$(MAKE) distclean;

$(FAKETARGETFORMAKE) : $(IRODSEXTERNALLOCAL)
	@echo "Unpacking $(IRODSEXTERNALLOCAL)..."
	$(V_at)tar zxf $(IRODSEXTERNALLOCAL)
	$(V_at)touch $(FAKETARGETFORMAKE)

ifeq ($(OS_platform), osx_platform)
build : cmake libarchive jansson boost epm hdf5 netcdf lib_mysqludf_preg libs3 check avro zeromq build_database_for_macosx
else
build : cmake libarchive jansson boost epm hdf5 netcdf lib_mysqludf_preg libs3 check avro zeromq
endif

generate : build
	@echo "Building $(IRODSEXTERNALARCHIVE)..."
	$(V_at)tar zfc $(IRODSEXTERNALARCHIVE) --exclude "*.tar.gz" --exclude "*.zip" --exclude "*cmake*" --exclude "Makefile" --exclude $(FAKETARGETFORMAKE) *
	$(V_at)cp $(IRODSEXTERNALARCHIVE) $(IRODSEXTERNALLOCAL)
	$(V_at)touch $(FAKETARGETFORMAKE)

$(LIB_MYSQLUDF_PREG_ARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(LIB_MYSQLUDF_PREG_ARCHIVE) 2>&1

$(LIB_MYSQLUDF_PREG_MAKE) : $(LIB_MYSQLUDF_PREG_ARCHIVE)
	@tar xf $(LIB_MYSQLUDF_PREG_ARCHIVE)
	@+cd lib_mysqludf_preg ; ./configure

$(LIB_MYSQLUDF_PREG) : $(LIB_MYSQLUDF_PREG_MAKE)
	@$(MAKE) -C $(LIB_MYSQLUDF_PREG)

$(LIBS3VERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(LIBS3VERSIONARCHIVE) 2>&1

$(LIBS3VERSIONMAKE) : $(LIBS3VERSIONARCHIVE)
	@tar zxf $(LIBS3VERSIONARCHIVE)

libs3 : $(LIBS3VERSIONMAKE)
	$(V_at)if [ $(OSNAME) = "MacOSX" ] ; then \
		CFLAGS=-fPIC $(MAKE) -C $(LIBS3VERSION) -f GNUmakefile.osx; \
		else \
		CFLAGS=-fPIC $(MAKE) -C $(LIBS3VERSION); \
		fi

$(CMAKEVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(CMAKEVERSIONARCHIVE) 2>&1

$(CMAKEVERSIONMAKE) : $(CMAKEVERSIONARCHIVE)
	@tar xf $(CMAKEVERSIONARCHIVE) # confusingly, cmake archive is not actually gzipped, just tarred
	@sed "s/std::cerr << \"loading initial cache file \"/std::cout << \"loading initial cache file \"/" $(CMAKEVERSION)/Source/cmake.cxx > /tmp/cmake.cxx
	@rsync -c /tmp/cmake.cxx $(CMAKEVERSION)/Source/cmake.cxx
	@rm -f /tmp/cmake.cxx
	@+cd $(CMAKEVERSION) ; \
		./bootstrap -- -DBUILD_TESTING=FALSE

cmake : $(CMAKEVERSIONMAKE)
	@$(MAKE) -C $(CMAKEVERSION)

$(CHECKVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(CHECKVERSIONARCHIVE) 2>&1

$(CHECKVERSIONMAKE) : $(CHECKVERSIONARCHIVE)
	@tar zxf $(CHECKVERSIONARCHIVE)
	@cd $(CHECKVERSION) ; \
		./configure --with-pic

check : $(CHECKVERSIONMAKE)
	@$(MAKE) -C $(CHECKVERSION)

$(JANSSONVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(JANSSONVERSIONARCHIVE) 2>&1

$(JANSSONVERSIONMAKE) : $(JANSSONVERSIONARCHIVE)
	@tar zxf $(JANSSONVERSIONARCHIVE)
	@cd $(JANSSONVERSION) ; \
		./configure --with-pic

jansson : $(JANSSONVERSIONMAKE)
	@$(MAKE) -C $(JANSSONVERSION)

$(LIBARCHIVEVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(LIBARCHIVEVERSIONARCHIVE) 2>&1

$(LIBARCHIVEVERSIONMAKE) : $(LIBARCHIVEVERSIONARCHIVE) cmake
	@tar zxf $(LIBARCHIVEVERSIONARCHIVE)
	@if [ $(OSNAME) != "MacOSX" ] ; then \
		bash -xc 'sed $(LIBARCHIVESEDCMD) $(LIBARCHIVEVERSION)/libarchive/test/test_sparse_basic.c > /tmp/test_sparse_basic.c' ; \
		rsync -c /tmp/test_sparse_basic.c $(LIBARCHIVEVERSION)/libarchive/test/test_sparse_basic.c ; \
		rm -f /tmp/test_sparse_basic.c ; \
		fi
	@sed "s/size_t used\;/size_t used = 0\;/" $(LIBARCHIVEVERSION)/libarchive/archive_read_support_format_xar.c > /tmp/archive_read_support_format_xar.c
	@rsync -c /tmp/archive_read_support_format_xar.c $(LIBARCHIVEVERSION)/libarchive/archive_read_support_format_xar.c
	@rm -f /tmp/archive_read_support_format_xar.c
	@cd $(LIBARCHIVEVERSION) ; \
		rm -f CMakeCache.txt ; \
		../$(CMAKEVERSION)/bin/cmake -D CMAKE_C_FLAGS:STRING=-fPIC .

libarchive : $(LIBARCHIVEVERSIONMAKE)
	@$(MAKE) -C $(LIBARCHIVEVERSION)

$(BOOSTVERSIONARCHIVE) :
	@wget $(RENCI_FTP_EXTERNAL)/$(BOOSTVERSIONARCHIVE) 2>&1

$(BOOSTVERSIONMAKE) : $(BOOSTVERSIONARCHIVE)
	@tar zxf $(BOOSTVERSIONARCHIVE)
	@cd $(BOOSTVERSION) ; \
		./bootstrap.sh --with-libraries=program_options,filesystem,system,thread,regex,program_options,iostreams,chrono,date_time

{'fraction_non_alphanumeric': 0.156022971009479, 'fraction_numerical': 0.007749256209783436, 'mean_word_length': 6.285282258064516, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1307917', 'n_tokens_mistral': 6025, 'n_tokens_neox': 5860, 'n_words': 834}