<commit_before>
# start here
cd osx/sources

# build icu and boost for packaging up within Mapnik Framework

# local install location
PREFIX=/Users/dane/projects/mapnik-dev/trunk-build/osx/sources
mkdir -p /Users/dane/projects/mapnik-dev/trunk-build/osx/sources
export DYLD_LIBRARY_PATH=$PREFIX/lib
# final resting place
INSTALL=/Library/Frameworks/Mapnik.framework/unix/lib
export DYLD_LIBRARY_PATH=$PREFIX/lib


# make a directory to hold icu and boost
mkdir -p ../deps
cd ../deps

# ICU
wget http://download.icu-project.org/files/icu4c/4.6/icu4c-4_6-src.tgz
tar xvf icu4c-4_6-src.tgz
cd icu/source

# universal flags
export CFLAGS="-O3 -arch i386 -arch x86_64"
export CXXFLAGS="-O3 -arch i386 -arch x86_64"
export LDFLAGS="-arch i386 -arch x86_64 -headerpad_max_install_names"
./runConfigureICU MacOSX --prefix=$PREFIX --disable-static --enable-shared --disable-samples --disable-icuio --disable-layout --disable-tests --disable-extras --with-library-bits=64
make install -j4
# note -R is needed to preserve the symlinks
#cp -R lib/libicuuc.* ../../../sources/lib/
#cp -R lib/libicudata.* ../../../sources/lib/
cd ../../../sources/lib/


# libicuuc
install_name_tool -id $INSTALL/libicuuc.46.dylib libicuuc.46.0.dylib
install_name_tool -change ../lib/libicudata.46.0.dylib $INSTALL/libicudata.46.dylib libicuuc.46.0.dylib
# libicudata
install_name_tool -id $INSTALL/libicudata.46.dylib libicudata.46.0.dylib
# libicui18n - needed by boost_regex
install_name_tool -id $INSTALL/libicui18n.46.dylib libicui18n.46.0.dylib
install_name_tool -change ../lib/libicudata.46.0.dylib $INSTALL/libicudata.46.dylib libicui18n.46.0.dylib
install_name_tool -change libicuuc.46.dylib $INSTALL/libicuuc.46.dylib libicui18n.46.0.dylib


wget http://cairographics.org/releases/pixman-0.21.4.tar.gz
tar xvf pixman-0.21.4.tar.gz
cd pixman-0.21.4
./configure --disable-dependency-tracking --prefix=$PREFIX
make -j4
make install

install_name_tool -id $INSTALL/libpixman-1.0.dylib ../../sources/lib/libpixman-1.0.dylib


# fontconfig
wget http://www.freedesktop.org/software/fontconfig/release/fontconfig-2.8.0.tar.gz
tar xvf fontconfig-2.8.0.tar.gz
cd fontconfig-2.8.0
./configure --disable-dependency-tracking --prefix=$PREFIX \
    --with-freetype-config=$PREFIX/bin/freetype-config
make -j4
make install
install_name_tool -id $INSTALL/libfontconfig.1.dylib ../../sources/lib/libfontconfig.1.dylib


# Cairo
wget http://cairographics.org/releases/cairo-1.10.2.tar.gz
tar xvf cairo-1.10.2.tar.gz
cd cairo-1.10.2
export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH
export LDFLAGS="-L/Library/Frameworks/UnixImageIO.framework/unix/lib "$LDFLAGS
export CFLAGS="-I/Library/Frameworks/UnixImageIO.framework/unix/include "$CFLAGS
export png_CFLAGS="-I/Library/Frameworks/UnixImageIO.framework/unix/include"
export png_LIBS="-I/Library/Frameworks/UnixImageIO.framework/unix/lib -lpng14"
./configure \
  --disable-valgrind \
  --enable-gobject=no \
  --enable-static=no \
  --enable-xlib=no \
  --enable-xlib-xrender=no \
  --enable-xcb=no \
  --enable-xlib-xcb=no \
  --enable-xcb-shm=no \
  --enable-xcb-drm=no \
  --disable-dependency-tracking \
  --prefix=$PREFIX
  
make -j4
make install

install_name_tool -id $INSTALL/libcairo.2.dylib ../../sources/lib/libcairo.2.dylib

# libsigcxx
wget http://ftp.gnome.org/pub/GNOME/sources/libsigc++/2.2/libsigc++-2.2.8.tar.gz
tar xvf libsigc++-2.2.8.tar.gz
cd libsigc++-2.2.8
./configure --disable-dependency-tracking --prefix=$PREFIX
make
make install

install_name_tool -id $INSTALL/libsigc-2.0.dylib ../../sources/lib/libsigc-2.0.dylib

wget http://cairographics.org/releases/cairomm-1.9.8.tar.gz
tar xvf cairomm-1.9.8.tar.gz
cd cairomm-1.9.8
export PKG_CONFIG_PATH=../../sources/lib/pkgconfig/
export LDFLAGS="-L$PREFIX/lib -lcairo -lsigc-2.0 "$LDFLAGS
export CFLAGS="-I$PREFIX/include -I$PREFIX/include/cairo -I$PREFIX/include/freetype2 -I$PREFIX/lib/sigc++-2.0/include -I$PREFIX/include/sigc++-2.0 -I$PREFIX/include/sigc++-2.0/sigc++ "$CFLAGS
export CXXFLAGS="-I$PREFIX/include "$CFLAGS

./configure --disable-dependency-tracking --prefix=$PREFIX
make -j4
make install

install_name_tool -id $INSTALL/libcairomm-1.0.1.dylib ../../sources/lib/libcairomm-1.0.1.dylib

# pycairo
# >= python 3.1
#wget http://cairographics.org/releases/pycairo-1.8.10.tar.bz2
#tar xvf pycairo-1.8.10.tar.bz2
#./waf configure

wget http://cairographics.org/releases/pycairo-1.8.8.tar.gz
tar xvf pycairo-1.8.8.tar.gz
cd pycairo-1.8.8
export PKG_CONFIG_PATH=../../sources/lib/pkgconfig/

# py25
# line 35 of configure.ac AM_PATH_PYTHON(2.5)
export PATH=/Library/Frameworks/Python.framework/Versions/2.5/bin/:$PATH
./configure --prefix=$PREFIX
make -j4 install

# py26
export PATH=/Library/Frameworks/Python.framework/Versions/2.6/bin/:$PATH
./configure --prefix=$PREFIX
make -j4 install

#py27
export PATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/:$PATH
make clean
./configure --prefix=$PREFIX
make -j4 install


# boost
cd ../../deps
wget http://voxel.dl.sourceforge.net/project/boost/boost/1.45.0/boost_1_45_0.tar.bz2
tar xjvf boost_1_45_0.tar.bz2
cd boost_1_45_0

# edit tools/build/v2/tools/python.jam, line 980, replace with:
    if $(target-os) in windows cygwin
    {
        alias python_for_extensions : python : $(target-requirements) ;
    }
    else if $(target-os) = darwin
    {
        alias python_for_extensions
            :
            : $(target-requirements)
            :
            : $(usage-requirements) <linkflags>"-undefined dynamic_lookup"
            ;
    }

./bootstrap.sh
#--prefix-dir
./bjam --prefix=$PREFIX --with-python --with-thread --with-filesystem \
  --with-iostreams --with-regex \
  --with-program_options --with-system \
  -sHAVE_ICU=1 -sICU_PATH=$PREFIX \
  toolset=darwin \
  address-model=32_64 \
  architecture=x86 \
  link=shared \
  stage

./bjam --prefix=$PREFIX --with-python --with-thread --with-filesystem \
  --with-iostreams --with-regex \
  --with-program_options --with-system \
  -sHAVE_ICU=1 -sICU_PATH=$PREFIX \
  toolset=darwin \
  address-model=32_64 \
  architecture=x86 \
  link=shared \
  install

# boost python for various versions are done in python script
python ../../scripts/build_boost_pythons.py 2.5 32_64
mv stage/lib/libboost_python.dylib stage/lib/libboost_python25.dylib
cp stage/lib/libboost_python25.dylib ../../sources/lib/libboost_python25.dylib

python ../../scripts/build_boost_pythons.py 2.6 32_64
mv stage/lib/libboost_python.dylib stage/lib/libboost_python26.dylib
cp stage/lib/libboost_python26.dylib ../../sources/lib/libboost_python26.dylib

python ../../scripts/build_boost_pythons.py 2.7 32_64
mv stage/lib/libboost_python.dylib stage/lib/libboost_python27.dylib
cp stage/lib/libboost_python27.dylib ../../sources/lib/libboost_python27.dylib

python ../../scripts/build_boost_pythons.py 3.1 32_64
cp stage/lib/libboost_python3.dylib ../../sources/lib/libboost_python31.dylib


#cp stage/lib/libboost_*dylib ../../sources/lib/

cd ../../sources/lib

# fix boost pythons
install_name_tool -id $INSTALL/libboost_python25.dylib libboost_python25.dylib
install_name_tool -id $INSTALL/libboost_python26.dylib libboost_python26.dylib
install_name_tool -id $INSTALL/libboost_python27.dylib libboost_python27.dylib
install_name_tool -id $INSTALL/libboost_python31.dylib libboost_python31.dylib

# fix boost libs
install_name_tool -id $INSTALL/libboost_system.dylib libboost_system.dylib
install_name_tool -id $INSTALL/libboost_filesystem.dylib libboost_filesystem.dylib
install_name_tool -id $INSTALL/libboost_regex.dylib libboost_regex.dylib
install_name_tool -id $INSTALL/libboost_program_options.dylib libboost_program_options.dylib
install_name_tool -id $INSTALL/libboost_iostreams.dylib libboost_iostreams.dylib
install_name_tool -id $INSTALL/libboost_thread.dylib libboost_thread.dylib
install_name_tool -change libboost_system.dylib $INSTALL/libboost_system.dylib libboost_filesystem.dylib
#install_name_tool -change libicui18n.46.dylib $INSTALL/libicui18n.46.dylib libboost_regex.dylib

# rasterlite we must bundle as it is not available in the SQLite.framework
cd ../../deps
svn co https://www.gaia-gis.it/svn/librasterlite
export LDFLAGS="-arch i386 -arch x86_64 -headerpad_max_install_names -L/Library/Frameworks/SQLite3.framework/unix/lib -L/Library/Frameworks/UnixImageIO.framework/unix/lib -L/Library/Frameworks/PROJ.framework/unix/lib"
export CFLAGS="-Os -arch i386 -arch x86_64 -I/Library/Frameworks/SQLite3.framework/unix/include -I/Library/Frameworks/UnixImageIO.framework/unix/include -I/Library/Frameworks/PROJ.framework/unix/include"
export CXXFLAGS=$CFLAGS
cd librasterlite
./configure --disable-dependency-tracking --prefix=$PREFIX
make clean
make -j4
make install

cd ../../sources/lib

install_name_tool -id $INSTALL/librasterlite.0.dylib librasterlite.0.dylib

# freetype2
cd ../../deps
wget http://download.savannah.gnu.org/releases/freetype/freetype-2.4.4.tar.gz
tar xvf freetype-2.4.4.tar.gz
export CFLAGS="-O3 -arch i386 -arch x86_64"
export LDFLAGS="-arch i386 -arch x86_64 -headerpad_max_install_names"
cd freetype-2.4.4
./configure --prefix=$PREFIX
make -j4
make install
cd ../../sources/lib
install_name_tool -id $INSTALL/libfreetype.6.dylib libfreetype.6.dylib

### MAPNIK ###

# make sure we set DYLD path so we can link to libs without installing
export DYLD_LIBRARY_PATH=$PREFIX/lib

# compile mapnik using osx/config.py
scons PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig

# then compile each python version..

# 2.5
rm bindings/python/*os<commit_msg>osx: update build scripts to boost 1.46.1 and python 3.2, dropping broken rasterlite<commit_after>
# start here
cd osx/sources

# build icu and boost for packaging up within Mapnik Framework

# local install location
PREFIX=/Users/dane/projects/mapnik-dev/trunk-build/osx/sources
mkdir -p /Users/dane/projects/mapnik-dev/trunk-build/osx/sources
export DYLD_LIBRARY_PATH=$PREFIX/lib
# final resting place
INSTALL=/Library/Frameworks/Mapnik.framework/unix/lib
export DYLD_LIBRARY_PATH=$PREFIX/lib
export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig


# make a directory to hold icu and boost
mkdir -p ../deps
cd ../deps

# ICU
wget http://download.icu-project.org/files/icu4c/4.6/icu4c-4_6-src.tgz
tar xvf icu4c-4_6-src.tgz
cd icu/source

# universal flags
export CFLAGS="-O3 -arch i386 -arch x86_64"
export CXXFLAGS="-O3 -arch i386 -arch x86_64"
export LDFLAGS="-arch i386 -arch x86_64 -headerpad_max_install_names"
#./runConfigureICU MacOSX --prefix=$PREFIX --disable-static --enable-shared --disable-samples --disable-icuio --disable-layout --disable-tests --disable-extras --with-library-bits=64
./runConfigureICU MacOSX --prefix=$PREFIX --disable-static --enable-shared --with-library-bits=64
make install -j4
# note -R is needed to preserve the symlinks
#cp -R lib/libicuuc.* ../../../sources/lib/
#cp -R lib/libicudata.* ../../../sources/lib/
cd ../../../sources/lib/


# libicuuc
install_name_tool -id $INSTALL/libicuuc.46.dylib libicuuc.46.0.dylib
install_name_tool -change ../lib/libicudata.46.0.dylib $INSTALL/libicudata.46.dylib libicuuc.46.0.dylib
# libicudata
install_name_tool -id $INSTALL/libicudata.46.dylib libicudata.46.0.dylib
# libicui18n - needed by boost_regex
install_name_tool -id $INSTALL/libicui18n.46.dylib libicui18n.46.0.dylib
install_name_tool -change ../lib/libicudata.46.0.dylib $INSTALL/libicudata.46.dylib libicui18n.46.0.dylib
install_name_tool -change libicuuc.46.dylib $INSTALL/libicuuc.46.dylib libicui18n.46.0.dylib
cd ../


# freetype2
wget http://download.savannah.gnu.org/releases/freetype/freetype-2.4.4.tar.gz
tar xvf freetype-2.4.4.tar.gz
export CFLAGS="-O3 -arch i386 -arch x86_64"
export LDFLAGS="-arch i386 -arch x86_64 -headerpad_max_install_names"
cd freetype-2.4.4
./configure --prefix=$PREFIX
make -j4
make install
install_name_tool -id $INSTALL/libfreetype.6.dylib ../../sources/lib/libfreetype.6.dylib
cd ../

# pixman
wget http://cairographics.org/releases/pixman-0.20.2.tar.gz
tar xvf pixman-0.20.2.tar.gz
cd pixman-0.20.2
./configure --disable-dependency-tracking --prefix=$PREFIX
make -j4
make install
cd ../../sources/lib/
install_name_tool -id $INSTALL/libpixman-1.0.dylib ../../sources/lib/libpixman-1.0.dylib
cd ../

# fontconfig
wget http://www.freedesktop.org/software/fontconfig/release/fontconfig-2.8.0.tar.gz
tar xvf fontconfig-2.8.0.tar.gz
cd fontconfig-2.8.0
./configure --disable-dependency-tracking --prefix=$PREFIX \
    --with-freetype-config=$PREFIX/bin/freetype-config
make -j4
make install
install_name_tool -id $INSTALL/libfontconfig.1.dylib ../../sources/lib/libfontconfig.1.dylib
cd ../

# Cairo
wget http://cairographics.org/releases/cairo-1.10.2.tar.gz
tar xvf cairo-1.10.2.tar.gz
cd cairo-1.10.2
export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH
export LDFLAGS="-L/Library/Frameworks/UnixImageIO.framework/unix/lib "$LDFLAGS
export CFLAGS="-I/Library/Frameworks/UnixImageIO.framework/unix/include "$CFLAGS
export png_CFLAGS="-I/Library/Frameworks/UnixImageIO.framework/unix/include"
export png_LIBS="-I/Library/Frameworks/UnixImageIO.framework/unix/lib -lpng14"
./configure \
  --disable-valgrind \
  --enable-gobject=no \
  --enable-static=no \
  --enable-xlib=no \
  --enable-xlib-xrender=no \
  --enable-xcb=no \
  --enable-xlib-xcb=no \
  --enable-xcb-shm=no \
  --enable-xcb-drm=no \
  --disable-dependency-tracking \
  --prefix=$PREFIX
  
make -j4
make install
install_name_tool -id $INSTALL/libcairo.2.dylib ../../sources/lib/libcairo.2.dylib
cd ../

# libsigcxx
wget http://ftp.gnome.org/pub/GNOME/sources/libsigc++/2.2/libsigc++-2.2.8.tar.gz
tar xvf libsigc++-2.2.8.tar.gz
cd libsigc++-2.2.8
./configure --disable-dependency-tracking --prefix=$PREFIX
make -j4
make install
install_name_tool -id $INSTALL/libsigc-2.0.dylib ../../sources/lib/libsigc-2.0.dylib
cd ../

wget http://cairographics.org/releases/cairomm-1.9.8.tar.gz
tar xvf cairomm-1.9.8.tar.gz
cd cairomm-1.9.8
export PKG_CONFIG_PATH=../../sources/lib/pkgconfig/
export LDFLAGS="-L$PREFIX/lib -lcairo -lsigc-2.0 "$LDFLAGS
export CFLAGS="-I$PREFIX/include -I$PREFIX/include/cairo -I$PREFIX/include/freetype2 -I$PREFIX/lib/sigc++-2.0/include -I$PREFIX/include/sigc++-2.0 -I$PREFIX/include/sigc++-2.0/sigc++ "$CFLAGS
export CXXFLAGS="-I$PREFIX/include "$CFLAGS

./configure --disable-dependency-tracking --prefix=$PREFIX
make -j4
make install

install_name_tool -id $INSTALL/libcairomm-1.0.1.dylib ../../sources/lib/libcairomm-1.0.1.dylib

# pycairo
# >= python 3.1
#wget http://cairographics.org/releases/pycairo-1.8.10.tar.bz2
#tar xvf pycairo-1.8.10.tar.bz2
#./waf configure

#wget http://cairographics.org/releases/pycairo-1.8.8.tar.gz
#tar xvf pycairo-1.8.8.tar.gz
#cd pycairo-1.8.8
#export PKG_CONFIG_PATH=../../sources/lib/pkgconfig/

# py25
# line 35 of configure.ac AM_PATH_PYTHON(2.5)
#export PATH=/Library/Frameworks/Python.framework/Versions/2.5/bin/:$PATH
#./configure --prefix=$PREFIX
#make -j4 install

# py26
#export PATH=/Library/Frameworks/Python.framework/Versions/2.6/bin/:$PATH
#./configure --prefix=$PREFIX
#make -j4 install

#py27
#export PATH=/Library/Frameworks/Python.framework/Versions/2.7/bin/:$PATH
#make clean
#./configure --prefix=$PREFIX
#make -j4 install


# boost
wget http://voxel.dl.sourceforge.net/project/boost/boost/1.46.1/boost_1_46_1.tar.bz2
tar xjvf boost_1_46_1.tar.bz2
cd boost_1_46_1

# edit tools/build/v2/tools/python.jam, line 980, replace with:
    if $(target-os) in windows cygwin
    {
        alias python_for_extensions : python : $(target-requirements) ;
    }
    else if $(target-os) = darwin
    {
        alias python_for_extensions
            :
            : $(target-requirements)
            :
            : $(usage-requirements) <linkflags>"-undefined dynamic_lookup"
            ;
    }

./bootstrap.sh
./bjam --prefix=$PREFIX --with-python --with-thread --with-filesystem \
  --with-regex --with-program_options --with-system \
  -sHAVE_ICU=1 -sICU_PATH=$PREFIX \
  toolset=darwin \
  address-model=32_64 \
  architecture=x86 \
  link=shared \
  variant=release \
  stage

./bjam --prefix=$PREFIX --with-python --with-thread --with-filesystem \
  --with-regex --with-program_options --with-system \
  -sHAVE_ICU=1 -sICU_PATH=$PREFIX \
  toolset=darwin \
  address-model=32_64 \
  architecture=x86 \
  link=shared \
  variant=release \
  install

# boost python for various versions are done in python script
python ../../scripts/build_boost_pythons.py 2.5 32_64
mv stage/lib/libboost_python.dylib stage/lib/libboost_python25.dylib
cp stage/lib/libboost_python25.dylib ../../sources/lib/libboost_python25.dylib

python ../../scripts/build_boost_pythons.py 2.6 32_64
mv stage/lib/libboost_python.dylib stage/lib/libboost_python26.dylib
cp stage/lib/libboost_python26.dylib ../../sources/lib/libboost_python26.dylib

python ../../scripts/build_boost_pythons.py 2.7 32_64
mv stage/lib/libboost_python.dylib stage/lib/libboost_python27.dylib
cp stage/lib/libboost_python27.dylib ../../sources/lib/libboost_python27.dylib

python ../../scripts/build_boost_pythons.py 3.1 32_64
mv stage/lib/libboost_python3.dylib stage/lib/libboost_python31.dylib
cp stage/lib/libboost_python31.dylib ../../sources/lib/libboost_python31.dylib

python ../../scripts/build_boost_pythons.py 3.2 32_64
mv stage/lib/libboost_python3.dylib stage/lib/libboost_python32.dylib
cp stage/lib/libboost_python32.dylib ../../sources/lib/libboost_python32.dylib


#cp stage/lib/libboost_*dylib ../../sources/lib/

cd ../../sources/lib

# fix boost pythons
install_name_tool -id $INSTALL/libboost_python25.dylib libboost_python25.dylib
install_name_tool -id $INSTALL/libboost_python26.dylib libboost_python26.dylib
install_name_tool -id $INSTALL/libboost_python27.dylib libboost_python27.dylib
install_name_tool -id $INSTALL/libboost_python31.dylib libboost_python31.dylib
install_name_tool -id $INSTALL/libboost_python32.dylib libboost_python32.dylib

# fix boost libs
install_name_tool -id $INSTALL/libboost_system.dylib libboost_system.dylib
install_name_tool -id $INSTALL/libboost_filesystem.dylib libboost_filesystem.dylib
install_name_tool -id $INSTALL/libboost_regex.dylib libboost_regex.dylib
install_name_tool -id $INSTALL/libboost_program_options.dylib libboost_program_options.dylib
install_name_tool -id $INSTALL/libboost_thread.dylib libboost_thread.dylib
install_name_tool -change libboost_system.dylib $INSTALL/libboost_system.dylib libboost_filesystem.dylib
#install_name_tool -change libicui18n.46.dylib $INSTALL/libicui18n.46.dylib libboost_regex.dylib

# currently broken, waiting on http://www.gaia-gis.it/rasterlite2/
# rasterlite we must bundle as it is not available in the SQLite.framework
#cd ../../deps
#svn co https://www.gaia-gis.it/svn/librasterlite
#export LDFLAGS="-arch i386 -arch x86_64 -headerpad_max_install_names -L/Library/Frameworks/SQLite3.framework/unix/lib -L/Library/Frameworks/UnixImageIO.framework/unix/lib -L/Library/Frameworks/PROJ.framework/unix/lib"
#export CFLAGS="-Os -arch i386 -arch x86_64 -I/Library/Frameworks/SQLite3.framework/unix/include -I/Library/Frameworks/UnixImageIO.framework/unix/include -I/Library/Frameworks/PROJ.framework/unix/include"
#export CXXFLAGS=$CFLAGS
#cd librasterlite
#./configure --disable-dependency-tracking --prefix=$PREFIX
#make clean
#make -j4
#make install
#cd ../../sources/lib
#install_name_tool -id $INSTALL/librasterlite.0.dylib librasterlite.0.dylib


### MAPNIK ###

# make sure we set DYLD path so we can link to libs without installing
export DYLD_LIBRARY_PATH=$PREFIX/lib

# compile mapnik using osx/config.py
scons PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig -j2 install BINDINGS=''

# then compile each python version..

# 2.5
rm bindings/python/*os
{'fraction_non_alphanumeric': 0.14080269264113418, 'fraction_numerical': 0.03865571931256056, 'mean_word_length': 3.8866184899077996, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5962878', 'n_tokens_mistral': 7975, 'n_tokens_neox': 7453, 'n_words': 1273}