<commit_before>#!/bin/bash
# vendored from https://github.com/h5py/h5py/blob/master/ci/get_hdf5_if_needed.sh

set -e

if [ -z ${HDF5_DIR+x} ]; then
    echo "Using OS HDF5"
else
    echo "Using downloaded HDF5"
    if [ -z ${HDF5_MPI+x} ]; then
        echo "Building serial"
        EXTRA_MPI_FLAGS=''
    else
        echo "Building with MPI"
        EXTRA_MPI_FLAGS="--enable-parallel --enable-shared"
    fi

    if [[ "$OSTYPE" == "darwin"* ]]; then
        lib_name=libhdf5.dylib
    else
        lib_name=libhdf5.so
    fi

    if [ -f $HDF5_DIR/lib/$lib_name ]; then
        echo "using cached build"
    else
        pushd /tmp
        #                                   Remove trailing .*, to get e.g. '1.12' ↓
        curl -fsSLO "https://www.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz"
        tar -xzvf hdf5-$HDF5_VERSION.tar.gz
        pushd hdf5-$HDF5_VERSION
        chmod u+x autogen.sh
        if [[ "${HDF5_VERSION%.*}" = "1.12" ]]; then
          ./configure --prefix $HDF5_DIR $EXTRA_MPI_FLAGS --enable-build-mode=production
        else
          ./configure --prefix $HDF5_DIR $EXTRA_MPI_FLAGS
        fi
        make -j $(nproc)
        make install
        popd
        popd
    fi<commit_msg>Build HDF5 as universal binary on osx.<commit_after>#!/bin/bash
# vendored from https://github.com/h5py/h5py/blob/master/ci/get_hdf5_if_needed.sh

set -e

if [ -z ${HDF5_DIR+x} ]; then
    echo "Using OS HDF5"
else
    echo "Using downloaded HDF5"
    if [ -z ${HDF5_MPI+x} ]; then
        echo "Building serial"
        EXTRA_MPI_FLAGS=''
    else
        echo "Building with MPI"
        EXTRA_MPI_FLAGS="--enable-parallel --enable-shared"
    fi

    MINOR_V=$(sed -e "s/.[0-9]\+\$//; s/^[0-9]\+.//" <<< $HDF5_VERSION)
    MAJOR_V=${HDF5_VERSION/%.*.*}
    if [[ "$OSTYPE" == "darwin"* ]]; then
        lib_name=libhdf5.dylib
        # universal binaries is only supported for v1.13+
        if [[ $MAJOR_V -gt 1 || $MINOR_V -ge 13 ]]; then
          export CFLAGS="$CFLAGS -arch x86_64 -arch arm64"
        else
    else
        lib_name=libhdf5.so
    fi

    if [ -f $HDF5_DIR/lib/$lib_name ]; then
        echo "using cached build"
    else
        pushd /tmp
        #                                   Remove trailing .*, to get e.g. '1.12' ↓
        curl -fsSLO "https://www.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz"
        tar -xzvf hdf5-$HDF5_VERSION.tar.gz
        pushd hdf5-$HDF5_VERSION
        chmod u+x autogen.sh
        # production is supported from 1.12
        if [[ $MAJOR_V -gt 1 || $MINOR_V -ge 12 ]]; then
          ./configure --prefix $HDF5_DIR $EXTRA_MPI_FLAGS --enable-build-mode=production 
        else
          ./configure --prefix $HDF5_DIR $EXTRA_MPI_FLAGS
        fi
        make -j $(nproc)
        make install
        file $HDF5_DIR/lib/$lib_name

        popd
        popd
    fi
{'fraction_non_alphanumeric': 0.13558178752107924, 'fraction_numerical': 0.027655986509274873, 'mean_word_length': 1.907843137254902, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4645484', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1112, 'n_words': 234}