<commit_before>#!/bin/bash -e

# Get the directory of where this script is located
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

# Output prefix
if [ -z "$PREFIX" ]
then
    export PREFIX="$SCRIPT_DIR/android-prefix"
fi

# Target platforms: 3, 4, 5, 8, 9, 14
PLATFORM=9

# Target architectures: arm, mips, x86
ARCH=arm

TOOLCHAIN=arm-linux-androideabi
TOOLCHAIN_VERSION=4.4.3
TOOLCHAIN_ARCH=linux-x86

SYSROOT="$ANDROID_NDK/platforms/android-$PLATFORM/arch-$ARCH/"

## For armv7 platforms
#FLAGS="-march=armv7-a -mfloat-abi=softfp"
#export CFLAGS="$FLAGS"
#export CXXFLAGS="$FLAGS"
#export LDFLAGS="-Wl,--fix-cortex-a8"

# Shouldn't need to change anything below here

TOOLCHAIN_PATH="$ANDROID_NDK/toolchains/$TOOLCHAIN-$TOOLCHAIN_VERSION/prebuilt/$TOOLCHAIN_ARCH/bin/"
export PATH="$TOOLCHAIN_PATH:$PATH"

export CC="$TOOLCHAIN-gcc --sysroot=$SYSROOT"
export CXX="$TOOLCHAIN-g++ --sysroot=$SYSROOT"
export LD="$TOOLCHAIN-ld --sysroot=$SYSROOT"

export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig"
export LD_LIBRARY_PATH="$PREFIX/lib"
export LDFLAGS="-L$PREFIX/lib"
export CPPFLAGS="-I$PREFIX/include"

BUILD_DIR="$SCRIPT_DIR/android-build"

AUTOMAKE_DIR=/usr/share/automake-1.12

UTIL_LINUX_BASE_VER=2.21
UTIL_LINUX_VER=2.21.2

EXPAT_VER=2.1.0

OPENSSL_VER=1.0.1c

if [ ! -d "$BUILD_DIR" ]
then
    mkdir -p "$BUILD_DIR"
    pushd "$BUILD_DIR"

    # util-linux, for libuuid
    if [ ! -d "util-linux-$UTIL_LINUX_VER" ]
    then
        wget -c "http://www.kernel.org/pub/linux/utils/util-linux/v$UTIL_LINUX_BASE_VER/util-linux-$UTIL_LINUX_VER.tar.xz"
        tar xf "util-linux-$UTIL_LINUX_VER.tar.xz"
        pushd "util-linux-$UTIL_LINUX_VER"

        # Need to update Autoconf's OS detection support for Android
        cp "$AUTOMAKE_DIR/config.guess" "$AUTOMAKE_DIR/config.sub" config/
        autoreconf -fi
        ./configure --host=arm-linux-androideabi --prefix="$PREFIX" \
            --disable-libblkid --disable-libmount --disable-mount \
            --disable-losetup --disable-fsck --disable-partx --disable-uuidd \
            --disable-mountpoint --disable-fallocate --disable-unshare \
            --disable-agetty --disable-cramfs --disable-switch_root \
            --disable-pivot_root --disable-kill --disable-rename \
            --disable-schedutils --disable-wall --disable-write \
            --without-ncurses --disable-shared
        pushd libuuid
        make $@
        make install
        popd
        popd
    fi

    # expat for XML parsing
    if [ ! -d "expat-$EXPAT_VER" ]
    then
        wget -c "http://downloads.sourceforge.net/project/expat/expat/$EXPAT_VER/expat-$EXPAT_VER.tar.gz"
        tar xf "expat-$EXPAT_VER.tar.gz"
        pushd "expat-$EXPAT_VER"
        ./configure --host=arm-linux-androideabi --prefix="$PREFIX" \
            --disable-shared
        make $@
        make install
        popd
    fi

    # openssl for SSL/TLS support
    if [ ! -d "openssl-$OPENSSL_VER" ]
    then
        wget -c "http://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz"
        tar xf "openssl-$OPENSSL_VER.tar.gz"
        pushd "openssl-$OPENSSL_VER"
        ANDROID_DEV="$SYSROOT/usr" ./Configure --prefix="$PREFIX" android -Wa,--noexecstack -DOPENSSL_NO_TLS1_2_CLIENT
        # The dependencies seem screwed up or something, so make fails the
        # first time...
        make deps
        make -k $@
        make $@
        make install
        popd
    fi

    popd
fi<commit_msg>Update build_android.sh deps script to be a bit cleaner.<commit_after>#!/bin/bash -e
# build_android.sh - Compiles dependencies for Android
#
# You may need to change some of the variables in this script for your
# system/target, specifically, PLATFORM, ARCH, TOOLCHAIN_VERSION, and
# AUTOMAKE_DIR.

if [ -z "$ANDROID_NDK" ]
then
    echo "Please set the ANDROID_NDK environment variable."
    exit 1
fi

# Get the directory of where this script is located
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

# Directory where the dependencies will be built
BUILD_DIR="$SCRIPT_DIR/android-build"

# Directory where the dependencies will be installed
PREFIX="$SCRIPT_DIR/android-prefix"

# Target platforms: 3, 4, 5, 8, 9, 14
PLATFORM=9

# Target architectures: arm, mips, x86
ARCH=arm

# Android NDK toolchain options
TOOLCHAIN=arm-linux-androideabi
# 4.6 is in NDK version 8b July 2012
TOOLCHAIN_VERSION=4.6

# Where to find config.guess and config.sub (needed to fix packages with old
# automake files)
AUTOMAKE_DIR="/usr/share/automake-1.12"

## For armv7 platforms
#FLAGS="-march=armv7-a -mfloat-abi=softfp"
#export CFLAGS="$FLAGS"
#export CXXFLAGS="$FLAGS"
#export LDFLAGS="-Wl,--fix-cortex-a8"

##############################################
# Shouldn't need to change anything below here
##############################################

# Modify TOOLCHAIN_ARCH based on host system
if [ "$(uname)" = "Linux" ]
then
    TOOLCHAIN_ARCH=linux-x86
elif [ "$(uname)" = "Darwin" ]
then
    TOOLCHAIN_ARCH=darwin-x86
fi

TOOLCHAIN_PATH="$ANDROID_NDK/toolchains/$TOOLCHAIN-$TOOLCHAIN_VERSION/prebuilt/$TOOLCHAIN_ARCH/bin/"
export PATH="$TOOLCHAIN_PATH:$PATH"

SYSROOT="$ANDROID_NDK/platforms/android-$PLATFORM/arch-$ARCH/"

#export CC="$TOOLCHAIN-gcc --sysroot=$SYSROOT -Wl,-rpath-link=$PREFIX/lib"
export CC="$TOOLCHAIN-gcc --sysroot=$SYSROOT"
export CXX="$TOOLCHAIN-g++ --sysroot=$SYSROOT"
export LD="$TOOLCHAIN-ld --sysroot=$SYSROOT"

export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig"
export LD_LIBRARY_PATH="$PREFIX/lib"
export LDFLAGS="-L$PREFIX/lib"
export CPPFLAGS="-I$PREFIX/include"

# Versions of dependent libraries
UTIL_LINUX_BASE_VER=2.21
UTIL_LINUX=util-linux-2.21.2
EXPAT_VER=2.1.0
EXPAT="expat-$EXPAT_VER"
OPENSSL=openssl-1.0.1c

# Simple function to download (if they haven't been already) and extract
# packages if they don't exist.
download_package() {
    if [ ! -f "$1" ]
    then
        curl -LO --disable-epsv "$2"
    fi
    tar -xf "$1"
}

mkdir -p "$BUILD_DIR"
pushd "$BUILD_DIR"

# util_linux - For libuuid
download_package "$UTIL_LINUX.tar.xz" "http://www.kernel.org/pub/linux/utils/util-linux/v$UTIL_LINUX_BASE_VER/$UTIL_LINUX.tar.xz"
pushd "$UTIL_LINUX"
## Need to update Autoconf's OS detection support for Android
cp "$AUTOMAKE_DIR/config.guess" "$AUTOMAKE_DIR/config.sub" config/
#autoreconf -fi
./configure --host=arm-linux-androideabi --prefix="$PREFIX" \
    --disable-libblkid --disable-libmount --disable-mount \
    --disable-losetup --disable-fsck --disable-partx --disable-uuidd \
    --disable-mountpoint --disable-fallocate --disable-unshare \
    --disable-agetty --disable-cramfs --disable-switch_root \
    --disable-pivot_root --disable-kill --disable-rename \
    --disable-schedutils --disable-wall --disable-write \
    --without-ncurses --disable-shared
pushd libuuid
make $@
make install
popd
popd

# Expat - For XML parsing
download_package "$EXPAT.tar.gz" "http://downloads.sourceforge.net/project/expat/expat/$EXPAT_VER/$EXPAT.tar.gz"
pushd "$EXPAT"
./configure --host=arm-linux-androideabi --prefix="$PREFIX" --disable-shared
make $@
make install
popd

# openssl for SSL/TLS support
download_package "$OPENSSL.tar.gz" "http://www.openssl.org/source/$OPENSSL.tar.gz"
pushd "$OPENSSL"
./Configure --prefix="$PREFIX" android no-idea no-cast no-seed no-md2 no-sha0 no-whrlpool no-whirlpool -DL_ENDIAN
# The dependencies seem screwed up or something, so make fails the
# first time...
make MAKEDEPPROG="$CC" depend
# Can't do parallel make for some reason.  It seems unreliable
make
make install
popd

popd
{'fraction_non_alphanumeric': 0.1482670948748829, 'fraction_numerical': 0.011508095811588384, 'mean_word_length': 3.4942874323511726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1510101', 'n_tokens_mistral': 2740, 'n_tokens_neox': 2586, 'n_words': 576}