<commit_before>#!/bin/bash
# vendored from https://github.com/h5py/h5py/blob/master/ci/get_hdf5_if_needed.sh

set -e -x

if [ -z ${HDF5_DIR+x} ]; then
    echo "Using OS HDF5"
else
    echo "Using downloaded HDF5"
    if [ -z ${HDF5_MPI+x} ]; then
        echo "Building serial"
        EXTRA_MPI_FLAGS=''
    else
        echo "Building with MPI"
        EXTRA_MPI_FLAGS="--enable-parallel --enable-shared"
    fi

    MINOR_V=${HDF5_VERSION#*.}
    MINOR_V=${MINOR_V%.*}
    MAJOR_V=${HDF5_VERSION/%.*.*}
    if [[ "$OSTYPE" == "darwin"* ]]; then
        lib_name=libhdf5.dylib
        NPROC=$(sysctl -n hw.ncpu)
    else
        lib_name=libhdf5.so
        NPROC=$(nproc)
    fi

    if [ -f "$HDF5_DIR/lib/$lib_name" ]; then
        echo "using cached build"
    else
        echo "building HDF5"
        if [[ "$OSTYPE" == "darwin"* ]]; then
            brew install automake cmake pkg-config
            export MACOSX_DEPLOYMENT_TARGET="10.9"
            pushd /tmp

            # lzo
            curl -sLO https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
            tar xzf lzo-2.10.tar.gz
            pushd lzo-2.10
            CFLAGS= CXXFLAGS= CPPFLAGS= CC="/usr/bin/clang" CXX="/usr/bin/clang" ./configure \
                --prefix="$HDF5_DIR"_arm64 CFLAGS="-arch arm64" CXXFLAGS="-arch arm64" \
                --host="aarch64-apple-darwin"  --target="aarch64-apple-darwin" --enable-shared
            make
            make install
            popd

            rm -rf lzo-2.10
            tar xzf lzo-2.10.tar.gz
            pushd lzo-2.10
            CFLAGS= CXXFLAGS= CPPFLAGS= CC="/usr/bin/clang" CXX="/usr/bin/clang" ./configure \
                --prefix="$HDF5_DIR"_x86 --enable-shared
            make
            make install
            popd
            
            mkdir -p "$HDF5_DIR"/lib
            pushd "$HDF5_DIR"_x86/lib
            for filename in *lzo*.dylib *lzo*.a; do
                if [[ -f "$HDF5_DIR"_arm64/lib/$filename ]]; then
                    lipo "$HDF5_DIR"_x86/lib/$filename "$HDF5_DIR"_arm64/lib/$filename -output "$HDF5_DIR"/lib/$filename -create
                fi
            done
            popd
            cp -r "$HDF5_DIR"_x86/include/lzo "$HDF5_DIR"/include/lzo
            cp "$HDF5_DIR"_x86/lib/pkgconfig/lzo2.pc "$HDF5_DIR"/lib/pkgconfig


            # snappy
            git clone https://github.com/google/snappy.git --branch 1.1.9 --depth 1
            pushd snappy
            git submodule update --init
            mkdir build
            cd build
            cmake -DCMAKE_INSTALL_PREFIX="$HDF5_DIR" -DENABLE_SHARED:bool=on -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" ../
            make
            make install
            popd

            # zstd
            curl -sLO https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz
            tar xzf zstd-1.5.2.tar.gz
            pushd zstd-1.5.2
            cd build/cmake
            cmake -DCMAKE_INSTALL_PREFIX="$HDF5_DIR" -DENABLE_SHARED:bool=on -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
            make
            make install
            popd<commit_msg>Include other missing dirs.<commit_after>#!/bin/bash
# vendored from https://github.com/h5py/h5py/blob/master/ci/get_hdf5_if_needed.sh

set -e -x

if [ -z ${HDF5_DIR+x} ]; then
    echo "Using OS HDF5"
else
    echo "Using downloaded HDF5"
    if [ -z ${HDF5_MPI+x} ]; then
        echo "Building serial"
        EXTRA_MPI_FLAGS=''
    else
        echo "Building with MPI"
        EXTRA_MPI_FLAGS="--enable-parallel --enable-shared"
    fi

    MINOR_V=${HDF5_VERSION#*.}
    MINOR_V=${MINOR_V%.*}
    MAJOR_V=${HDF5_VERSION/%.*.*}
    if [[ "$OSTYPE" == "darwin"* ]]; then
        lib_name=libhdf5.dylib
        NPROC=$(sysctl -n hw.ncpu)
    else
        lib_name=libhdf5.so
        NPROC=$(nproc)
    fi

    if [ -f "$HDF5_DIR/lib/$lib_name" ]; then
        echo "using cached build"
    else
        echo "building HDF5"
        if [[ "$OSTYPE" == "darwin"* ]]; then
            brew install automake cmake pkg-config
            export MACOSX_DEPLOYMENT_TARGET="10.9"
            pushd /tmp

            # lzo
            curl -sLO https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
            tar xzf lzo-2.10.tar.gz
            pushd lzo-2.10
            CFLAGS= CXXFLAGS= CPPFLAGS= CC="/usr/bin/clang" CXX="/usr/bin/clang" ./configure \
                --prefix="$HDF5_DIR"_arm64 CFLAGS="-arch arm64" CXXFLAGS="-arch arm64" \
                --host="aarch64-apple-darwin"  --target="aarch64-apple-darwin" --enable-shared
            make
            make install
            popd

            rm -rf lzo-2.10
            tar xzf lzo-2.10.tar.gz
            pushd lzo-2.10
            CFLAGS= CXXFLAGS= CPPFLAGS= CC="/usr/bin/clang" CXX="/usr/bin/clang" ./configure \
                --prefix="$HDF5_DIR"_x86 --enable-shared
            make
            make install
            popd
            
            mkdir -p "$HDF5_DIR"/lib/pkgconfig
            mkdir "$HDF5_DIR"/include
            pushd "$HDF5_DIR"_x86/lib
            for filename in *lzo*.dylib *lzo*.a; do
                if [[ -f "$HDF5_DIR"_arm64/lib/$filename ]]; then
                    lipo "$HDF5_DIR"_x86/lib/$filename "$HDF5_DIR"_arm64/lib/$filename -output "$HDF5_DIR"/lib/$filename -create
                fi
            done
            popd
            cp -r "$HDF5_DIR"_x86/include/lzo "$HDF5_DIR"/include/lzo
            cp "$HDF5_DIR"_x86/lib/pkgconfig/lzo2.pc "$HDF5_DIR"/lib/pkgconfig


            # snappy
            git clone https://github.com/google/snappy.git --branch 1.1.9 --depth 1
            pushd snappy
            git submodule update --init
            mkdir build
            cd build
            cmake -DCMAKE_INSTALL_PREFIX="$HDF5_DIR" -DENABLE_SHARED:bool=on -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" ../
            make
            make install
            popd

            # zstd
            curl -sLO https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz
            tar xzf zstd-1.5.2.tar.gz
            pushd zstd-1.5.2
            cd build/cmake
            cmake -DCMAKE_INSTALL_PREFIX="$HDF5_DIR" -DENABLE_SHARED:bool=on -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
            make
            make install
            popd
{'fraction_non_alphanumeric': 0.12543664655446174, 'fraction_numerical': 0.0322324547475389, 'mean_word_length': 1.6213066999583854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5913884', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2202, 'n_words': 402}