<commit_before>#!/usr/bin/env bash

set -eu
set -o pipefail

: '

todo

- docs for base setup: sudo apt-get -y install zlib1g-dev make git
- shrink icu data
'

MASON_VERSION="new-pkgs"

function setup_mason() {
    if [[ ! -d ./.mason ]]; then
        git clone https://github.com/mapbox/mason.git ./.mason
        (cd ./.mason && git checkout ${MASON_VERSION})
    else
        echo "Updating to latest mason"
        (cd ./.mason && git fetch && git checkout ${MASON_VERSION} && git pull)
    fi
    export PATH=$(pwd)/.mason:$PATH
    export CXX=${CXX:-clang++}
    export CC=${CC:-clang}
}

function install() {
    MASON_PLATFORM_ID=$(mason env MASON_PLATFORM_ID)
    if [[ ! -d ./mason_packages/${MASON_PLATFORM_ID}/${1}/${2} ]]; then
        mason install $1 $2
        if [[ ${3:-false} != false ]]; then
            LA_FILE=$(mason prefix $1 $2)/lib/$3.la
            if [[ -f ${LA_FILE} ]]; then
                perl -i -p -e 's:\Q$ENV{HOME}/build/mapbox/mason\E:$ENV{PWD}:g' ${LA_FILE}
            else
                echo "$LA_FILE not found"
            fi
        fi
    fi
    # the rm here is to workaround https://github.com/mapbox/mason/issues/230
    rm -f ./mason_packages/.link/mason.ini
    mason link $1 $2
}

ICU_VERSION="55.1"

function install_mason_deps() {
    FAIL=0
    install ccache 3.3.0 &
    install zlib system &
    install jpeg_turbo 1.5.0 libjpeg &
    install libpng 1.6.24 libpng &
    install libtiff 4.0.6 libtiff &
    install libpq 9.5.2 &
    install sqlite 3.14.1 libsqlite3 &
    install expat 2.2.0 libexpat &
    install icu ${ICU_VERSION} &
    install proj 4.9.2 libproj &
    install pixman 0.34.0 libpixman-1 &
    install cairo 1.14.6 libcairo &
    install protobuf 2.6.1 &
    # technically protobuf is not a mapnik core dep, but installing
    # here by default helps make mapnik-vector-tile builds easier
    install webp 0.5.1 libwebp &
    install gdal 2.1.1 libgdal &
    install boost 1.61.0 &
    install boost_libsystem 1.61.0 &
    install boost_libfilesystem 1.61.0 &
    install boost_libprogram_options 1.61.0 &
    install boost_libregex_icu 1.61.0 &
    # technically boost thread and python are not a core dep, but installing
    # here by default helps make python-mapnik builds easier
    install boost_libthread 1.61.0 &
    install boost_libpython 1.61.0 &
    install freetype 2.6.5 libfreetype &
    install harfbuzz 1.3.0 libharfbuzz &
    for job in $(jobs -p)
    do
        wait $job || let "FAIL+=1"
    done
    if [[ "$FAIL" != "0" ]]; then
        exit ${FAIL}
    fi
}

MASON_LINKED_ABS=$(pwd)/mason_packages/.link
MASON_LINKED_REL=./mason_packages/.link
export C_INCLUDE_PATH="${MASON_LINKED_ABS}/include"
export CPLUS_INCLUDE_PATH="${MASON_LINKED_ABS}/include"
export LIBRARY_PATH="${MASON_LINKED_ABS}/lib"

function make_config() {
    echo "
CXX = '$CXX'
CC = '$CC'
RUNTIME_LINK = 'static'
INPUT_PLUGINS = 'all'
PATH = '${MASON_LINKED_REL}/bin'
PKG_CONFIG_PATH = '${MASON_LINKED_REL}/lib/pkgconfig'
PATH_REMOVE = '/usr:/usr/local'<commit_msg>mason packages are currently built with -D_GLIBCXX_USE_CXX11_ABI=0<commit_after>#!/usr/bin/env bash

set -eu
set -o pipefail

: '

todo

- docs for base setup: sudo apt-get -y install zlib1g-dev make git
- shrink icu data
'

MASON_VERSION="new-pkgs"

function setup_mason() {
    if [[ ! -d ./.mason ]]; then
        git clone https://github.com/mapbox/mason.git ./.mason
        (cd ./.mason && git checkout ${MASON_VERSION})
    else
        echo "Updating to latest mason"
        (cd ./.mason && git fetch && git checkout ${MASON_VERSION} && git pull)
    fi
    export PATH=$(pwd)/.mason:$PATH
    export CXX=${CXX:-clang++}
    export CC=${CC:-clang}
}

function install() {
    MASON_PLATFORM_ID=$(mason env MASON_PLATFORM_ID)
    if [[ ! -d ./mason_packages/${MASON_PLATFORM_ID}/${1}/${2} ]]; then
        mason install $1 $2
        if [[ ${3:-false} != false ]]; then
            LA_FILE=$(mason prefix $1 $2)/lib/$3.la
            if [[ -f ${LA_FILE} ]]; then
                perl -i -p -e 's:\Q$ENV{HOME}/build/mapbox/mason\E:$ENV{PWD}:g' ${LA_FILE}
            else
                echo "$LA_FILE not found"
            fi
        fi
    fi
    # the rm here is to workaround https://github.com/mapbox/mason/issues/230
    rm -f ./mason_packages/.link/mason.ini
    mason link $1 $2
}

ICU_VERSION="55.1"

function install_mason_deps() {
    FAIL=0
    install ccache 3.3.0 &
    install zlib system &
    install jpeg_turbo 1.5.0 libjpeg &
    install libpng 1.6.24 libpng &
    install libtiff 4.0.6 libtiff &
    install libpq 9.5.2 &
    install sqlite 3.14.1 libsqlite3 &
    install expat 2.2.0 libexpat &
    install icu ${ICU_VERSION} &
    install proj 4.9.2 libproj &
    install pixman 0.34.0 libpixman-1 &
    install cairo 1.14.6 libcairo &
    install protobuf 2.6.1 &
    # technically protobuf is not a mapnik core dep, but installing
    # here by default helps make mapnik-vector-tile builds easier
    install webp 0.5.1 libwebp &
    install gdal 2.1.1 libgdal &
    install boost 1.61.0 &
    install boost_libsystem 1.61.0 &
    install boost_libfilesystem 1.61.0 &
    install boost_libprogram_options 1.61.0 &
    install boost_libregex_icu 1.61.0 &
    # technically boost thread and python are not a core dep, but installing
    # here by default helps make python-mapnik builds easier
    install boost_libthread 1.61.0 &
    install boost_libpython 1.61.0 &
    install freetype 2.6.5 libfreetype &
    install harfbuzz 1.3.0 libharfbuzz &
    for job in $(jobs -p)
    do
        wait $job || let "FAIL+=1"
    done
    if [[ "$FAIL" != "0" ]]; then
        exit ${FAIL}
    fi
}

MASON_LINKED_ABS=$(pwd)/mason_packages/.link
MASON_LINKED_REL=./mason_packages/.link
export C_INCLUDE_PATH="${MASON_LINKED_ABS}/include"
export CPLUS_INCLUDE_PATH="${MASON_LINKED_ABS}/include"
export LIBRARY_PATH="${MASON_LINKED_ABS}/lib"

function make_config() {
    echo "
CXX = '$CXX'
CC = '$CC'
CUSTOM_CXXFLAGS = '-D_GLIBCXX_USE_CXX11_ABI=0'
RUNTIME_LINK = 'static'
INPUT_PLUGINS = 'all'
PATH = '${MASON_LINKED_REL}/bin'
PKG_CONFIG_PATH = '${MASON_LINKED_REL}/lib/pkgconfig'
PATH_REMOVE = '/usr:/usr/local'
{'fraction_non_alphanumeric': 0.12726977950713358, 'fraction_numerical': 0.033073929961089495, 'mean_word_length': 2.676400476758045, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1913344', 'n_tokens_mistral': 2397, 'n_tokens_neox': 2273, 'n_words': 571}