<commit_before>#!/bin/bash
#------------------------------------------------------------------------------
#
# Python for android
# https://github.com/tito/python-for-android
#
#------------------------------------------------------------------------------

# Modules
MODULES=

# Resolve Python path
PYTHON="$(which python2.7)"
if [ "X$PYTHON" == "X" ]; then
	PYTHON="$(which python2)"
fi
if [ "X$PYTHON" == "X" ]; then
	PYTHON="$(which python)"
fi

# Paths
ROOT_PATH="$(dirname $($PYTHON -c 'from __future__ import print_function; import os,sys;print(os.path.realpath(sys.argv[1]))' $0))"
RECIPES_PATH="$ROOT_PATH/recipes"
BUILD_PATH="$ROOT_PATH/build"
LIBS_PATH="$ROOT_PATH/build/libs"
JAVACLASS_PATH="$ROOT_PATH/build/java"
PACKAGES_PATH="$ROOT_PATH/.packages"
SRC_PATH="$ROOT_PATH/src"
JNI_PATH="$SRC_PATH/jni"
DIST_PATH="$ROOT_PATH/dist/default"

# Tools
export LIBLINK_PATH="$BUILD_PATH/objects"
export LIBLINK="$ROOT_PATH/src/tools/liblink"
export BIGLINK="$ROOT_PATH/src/tools/biglink"
export PIP="pip-2.7"

MD5SUM=$(which md5sum)
if [ "X$MD5SUM" == "X" ]; then
	MD5SUM=$(which md5)
	if [ "X$MD5SUM" == "X" ]; then
		echo "Error: you need at least md5sum or md5 installed."
		exit 1
	else
		MD5SUM="$MD5SUM -r"
	fi
fi

WGET=$(which wget)
if [ "X$WGET" == "X" ]; then
	WGET=$(which curl)
	if [ "X$WGET" == "X" ]; then
		echo "Error: you need at least wget or curl installed."
		exit 1
	else
		WGET="$WGET -L -O -o"
	fi
else
	WGET="$WGET -O"
fi

case $OSTYPE in
	darwin*)
		SED="sed -i ''"
		;;
	*)
		SED="sed -i"
		;;
esac


# Internals
CRED="\x1b[31;01m"
CBLUE="\x1b[34;01m"
CGRAY="\x1b[30;01m"
CRESET="\x1b[39;49;00m"
DO_CLEAN_BUILD=0
DO_SET_X=0

# Use ccache ?
which ccache &>/dev/null
if [ $? -eq 0 ]; then
	export CC="ccache gcc"
	export CXX="ccache g++"
	export NDK_CCACHE="ccache"
fi

function try () {
    "$@" || exit -1
}

function info() {
	echo -e "$CBLUE"$@"$CRESET";
}

function error() {
	echo -e "$CRED"$@"$CRESET";
}

function debug() {
	echo -e "$CGRAY"$@"$CRESET";
}

function get_directory() {
	case $1 in
		*.tar.gz)	directory=$(basename $1 .tar.gz) ;;
		*.tgz)		directory=$(basename $1 .tgz) ;;
		*.tar.bz2)	directory=$(basename $1 .tar.bz2) ;;
		*.tbz2)		directory=$(basename $1 .tbz2) ;;
		*.zip)		directory=$(basename $1 .zip) ;;
		*)
			error "Unknown file extension $1"
			exit -1
			;;
	esac
	echo $directory
}

function push_arm() {
	info "Entering in ARM enviromnent"

	# save for pop
	export OLD_PATH=$PATH
	export OLD_CFLAGS=$CFLAGS
	export OLD_CXXFLAGS=$CXXFLAGS
	export OLD_LDFLAGS=$LDFLAGS
	export OLD_CC=$CC
	export OLD_CXX=$CXX
	export OLD_AR=$AR
	export OLD_RANLIB=$RANLIB
	export OLD_STRIP=$STRIP
	export OLD_MAKE=$MAKE
	export OLD_LD=$LD

	# to override the default optimization, set OFLAG
	#export OFLAG="-Os"
	#export OFLAG="-O2"

	export CFLAGS="-DANDROID -mandroid $OFLAG -fomit-frame-pointer --sysroot $NDKPLATFORM"
	if [ "X$ARCH" == "Xarmeabi-v7a" ]; then
		CFLAGS+=" -march=armv7-a -mfloat-abi=softfp -mfpu=vfp -mthumb"
	fi
	export CXXFLAGS="$CFLAGS"

	# that could be done only for darwin platform, but it doesn't hurt.
	export LDFLAGS="-lm"

	# this must be something depending of the API level of Android
	PYPLATFORM=$($PYTHON -c 'from __future__ import print_function; import sys; print(sys.platform)')
	if [ "$PYPLATFORM" == "linux2" ]; then
		PYPLATFORM="linux"
	elif [ "$PYPLATFORM" == "linux3" ]; then
		PYPLATFORM="linux"
	fi

    if [ "X$ANDROIDNDKVER" == "Xr5b" ]; then
        export TOOLCHAIN_PREFIX=arm-eabi
        export TOOLCHAIN_VERSION=4.4.0
    elif [ "X${ANDROIDNDKVER:0:2}" == "Xr7" ] || [ "X${ANDROIDNDKVER:0:2}" == "Xr8" ]; then
        export TOOLCHAIN_PREFIX=arm-linux-androideabi
        export TOOLCHAIN_VERSION=4.4.3
    elif  [ "X${ANDROIDNDKVER}" == "Xr9" ]; then
            export TOOLCHAIN_PREFIX=arm-linux-androideabi
            export TOOLCHAIN_VERSION=4.8<commit_msg>check for r9xx and raise error for newer ndk<commit_after>#!/bin/bash
#------------------------------------------------------------------------------
#
# Python for android
# https://github.com/tito/python-for-android
#
#------------------------------------------------------------------------------

# Modules
MODULES=

# Resolve Python path
PYTHON="$(which python2.7)"
if [ "X$PYTHON" == "X" ]; then
	PYTHON="$(which python2)"
fi
if [ "X$PYTHON" == "X" ]; then
	PYTHON="$(which python)"
fi

# Paths
ROOT_PATH="$(dirname $($PYTHON -c 'from __future__ import print_function; import os,sys;print(os.path.realpath(sys.argv[1]))' $0))"
RECIPES_PATH="$ROOT_PATH/recipes"
BUILD_PATH="$ROOT_PATH/build"
LIBS_PATH="$ROOT_PATH/build/libs"
JAVACLASS_PATH="$ROOT_PATH/build/java"
PACKAGES_PATH="$ROOT_PATH/.packages"
SRC_PATH="$ROOT_PATH/src"
JNI_PATH="$SRC_PATH/jni"
DIST_PATH="$ROOT_PATH/dist/default"

# Tools
export LIBLINK_PATH="$BUILD_PATH/objects"
export LIBLINK="$ROOT_PATH/src/tools/liblink"
export BIGLINK="$ROOT_PATH/src/tools/biglink"
export PIP="pip-2.7"

MD5SUM=$(which md5sum)
if [ "X$MD5SUM" == "X" ]; then
	MD5SUM=$(which md5)
	if [ "X$MD5SUM" == "X" ]; then
		echo "Error: you need at least md5sum or md5 installed."
		exit 1
	else
		MD5SUM="$MD5SUM -r"
	fi
fi

WGET=$(which wget)
if [ "X$WGET" == "X" ]; then
	WGET=$(which curl)
	if [ "X$WGET" == "X" ]; then
		echo "Error: you need at least wget or curl installed."
		exit 1
	else
		WGET="$WGET -L -O -o"
	fi
else
	WGET="$WGET -O"
fi

case $OSTYPE in
	darwin*)
		SED="sed -i ''"
		;;
	*)
		SED="sed -i"
		;;
esac


# Internals
CRED="\x1b[31;01m"
CBLUE="\x1b[34;01m"
CGRAY="\x1b[30;01m"
CRESET="\x1b[39;49;00m"
DO_CLEAN_BUILD=0
DO_SET_X=0

# Use ccache ?
which ccache &>/dev/null
if [ $? -eq 0 ]; then
	export CC="ccache gcc"
	export CXX="ccache g++"
	export NDK_CCACHE="ccache"
fi

function try () {
    "$@" || exit -1
}

function info() {
	echo -e "$CBLUE"$@"$CRESET";
}

function error() {
	echo -e "$CRED"$@"$CRESET";
}

function debug() {
	echo -e "$CGRAY"$@"$CRESET";
}

function get_directory() {
	case $1 in
		*.tar.gz)	directory=$(basename $1 .tar.gz) ;;
		*.tgz)		directory=$(basename $1 .tgz) ;;
		*.tar.bz2)	directory=$(basename $1 .tar.bz2) ;;
		*.tbz2)		directory=$(basename $1 .tbz2) ;;
		*.zip)		directory=$(basename $1 .zip) ;;
		*)
			error "Unknown file extension $1"
			exit -1
			;;
	esac
	echo $directory
}

function push_arm() {
	info "Entering in ARM enviromnent"

	# save for pop
	export OLD_PATH=$PATH
	export OLD_CFLAGS=$CFLAGS
	export OLD_CXXFLAGS=$CXXFLAGS
	export OLD_LDFLAGS=$LDFLAGS
	export OLD_CC=$CC
	export OLD_CXX=$CXX
	export OLD_AR=$AR
	export OLD_RANLIB=$RANLIB
	export OLD_STRIP=$STRIP
	export OLD_MAKE=$MAKE
	export OLD_LD=$LD

	# to override the default optimization, set OFLAG
	#export OFLAG="-Os"
	#export OFLAG="-O2"

	export CFLAGS="-DANDROID -mandroid $OFLAG -fomit-frame-pointer --sysroot $NDKPLATFORM"
	if [ "X$ARCH" == "Xarmeabi-v7a" ]; then
		CFLAGS+=" -march=armv7-a -mfloat-abi=softfp -mfpu=vfp -mthumb"
	fi
	export CXXFLAGS="$CFLAGS"

	# that could be done only for darwin platform, but it doesn't hurt.
	export LDFLAGS="-lm"

	# this must be something depending of the API level of Android
	PYPLATFORM=$($PYTHON -c 'from __future__ import print_function; import sys; print(sys.platform)')
	if [ "$PYPLATFORM" == "linux2" ]; then
		PYPLATFORM="linux"
	elif [ "$PYPLATFORM" == "linux3" ]; then
		PYPLATFORM="linux"
	fi

    if [ "X$ANDROIDNDKVER" == "Xr5b" ]; then
        export TOOLCHAIN_PREFIX=arm-eabi
        export TOOLCHAIN_VERSION=4.4.0
    elif [ "X${ANDROIDNDKVER:0:2}" == "Xr7" ] || [ "X${ANDROIDNDKVER:0:2}" == "Xr8" ]; then
        export TOOLCHAIN_PREFIX=arm-linux-androideabi
        export TOOLCHAIN_VERSION=4.4.3
    elif  [ "X${ANDROIDNDKVER:0:2}" == "Xr9" ]; then
            export TOOLCHAIN_PREFIX=arm-linux-androideabi
            export TOOLCHAIN_VERSION=4.8
    else
        echo "Error: Please report issue to enable support for newer ndk."
        exit 1
{'fraction_non_alphanumeric': 0.2012226184411615, 'fraction_numerical': 0.020376974019358125, 'mean_word_length': 3.3920581655480984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2301442', 'n_tokens_mistral': 3293, 'n_tokens_neox': 3131, 'n_words': 697}