<commit_before>#!/bin/bash
# PARAMS:
# $1: Target ID (for example mb526).
# $2: "Name" of the ROM (this will be used as folder in the ROM_OUTPUT_DIRECTORY).
# PWD: This script has to be started from the "source code" folder.

BIN_DIR=${BIN_DIR:-"/home/android/"}
ROM_DATABASE_SCRIPT_DIR=${ROM_DATABASE_SCRIPT_DIR:-"${BIN_DIR}/cm-update-api/"}

DEVICE_ID=${DEVICE_ID:-"$1"}
ROM_SUBDIRECTORY=${ROM_SUBDIRECTORY:-"$2"}
LOCAL_MANIFEST=${LOCAL_MANIFEST:-"$3"}

if [[ -z "${DEVICE_ID}" ]]
then
	echo "Argument #1 (device ID) is required!"
	exit 1
fi

if [[ -z "${ROM_SUBDIRECTORY}" ]]
then
	echo "Argument #2 (ROM subdirectory) is required!"
	exit 1
fi

echo "Build-script was called for target '${DEVICE_ID}' and result directory '${ROM_SUBDIRECTORY}'."

PUBLIC_ROM_DIRECTORY=${PUBLIC_ROM_DIRECTORY:-"${BIN_DIR}/roms/${ROM_SUBDIRECTORY}"}
INCREMENTAL_UPDATES_DIRECTORY=${INCREMENTAL_UPDATES_DIRECTORY:-"${BIN_DIR}/incrementals/${ROM_SUBDIRECTORY}"}
TARGET_FILES_DIRECTORY=${TARGET_FILES_DIRECTORY:-"${BIN_DIR}/targetfiles/${ROM_SUBDIRECTORY}"}

ROM_OUTPUT_DIR="out/target/product/${DEVICE_ID}/"
TARGET_FILES_OUTPUT_DIR="${ROM_OUTPUT_DIR}/obj/PACKAGING/target_files_intermediates/"

DELETE_ROMS_OLDER_THAN=${DELETE_ROMS_OLDER_THAN:-7}

function disable_build {
	FILENAME="${1}"

	(cd $ROM_DATABASE_SCRIPT_DIR && \
		node disable-build.js \
			--device $DEVICE_ID \
			--filename $FILENAME \
			--subdirectory $ROM_SUBDIRECTORY \
			--disable_incrementals)
}

set -e
set -o xtrace
set -o pipefail

# Building will fail of no valid locale is set.
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
export LANGUAGE=en_US.UTF-8

# Let's make sure JAVA_HOME is in our path and found first.
if [ ! -z "${JAVA_HOME}" ]
then
	echo "Using JDK from '${JAVA_HOME}'."
	export PATH="${JAVA_HOME}/bin:${PATH}"
fi

if [ -n "${LOCAL_MANIFEST}" ]
then
	echo "Using local manifest: ${LOCAL_MANIFEST}"

	if [ -e .repo/local_manifests/*.xml ]
	then
		rm .repo/local_manifests/*.xml
	fi

	SOURCE_MANIFEST_FILEPATH=$(readlink -e "./.repo/manifests/${LOCAL_MANIFEST}")
	ln -s "${SOURCE_MANIFEST_FILEPATH}" .repo/local_manifests/
fi

export USE_CCACHE=${USE_CCACHE:-1}

if [[ -z "${CCACHE_DIR}" ]]
then
	export CCACHE_DIR="${BIN_DIR}/.ccache/"
fi

if [[ "${SKIP_REPO_SYNC}" == "true" ]]
then
	echo "Skipping 'repo sync'"
else
	SOURCE_TIMESTAMP=$(date +"%s")

	echo "Starting 'repo sync'..."
	time repo sync
fi

export CM_BUILDTYPE=${CM_BUILDTYPE:-NIGHTLY}

echo "Getting CM prebuilts..."
time vendor/cm/get-prebuilts

echo "Setting up build environment..."
. ./build/envsetup.sh

IONICE="ionice -c3"
SCHEDULING="${IONICE} schedtool -B -n19 -e"

if [[ "${SKIP_MAKE_CLEAN}" == "true" ]]
then
	echo "Skipping make clean"

	# Remove potentially stale files.
	rm -f $ROM_OUTPUT_DIR/cm-*.zip
	rm -f $ROM_OUTPUT_DIR/incremental*.zip
	rm -f $ROM_OUTPUT_DIR/md5sum
	rm -f $TARGET_FILES_OUTPUT_DIR/cm_*.zip
	rm -f $ROM_OUTPUT_DIR/system/build.prop
else
	echo "Cleaning output directory..."
	time $SCHEDULING make -j1 clean
fi

# Required, because otherwise CM won't build.
set +e
set +o xtrace

echo "Configuring build..."
time breakfast "${DEVICE_ID}" || exit 1

echo "Starting build..."
time $SCHEDULING make -j1 bacon || exit 1

echo "Finished build!"

set -e
set -o xtrace

TARGET_ROM_ZIP=$ROM_OUTPUT_DIR/cm-*.zip
TARGET_ROM_MD5SUM=$TARGET_ROM_ZIP.md5sum
TARGET_FILES_ZIP=$TARGET_FILES_OUTPUT_DIR/cm_*target*.zip

# Don't use absolute paths in the md5sum file.
sed -r -i "s|$(readlink -e ${ROM_OUTPUT_DIR})/?||g" $TARGET_ROM_MD5SUM

ls -la ${PUBLIC_ROM_DIRECTORY}

echo "Removing builds older than ${DELETE_ROMS_OLDER_THAN} days..."

# Remove old builds
for FILE in $(find "${PUBLIC_ROM_DIRECTORY}" -type f -mtime +${DELETE_ROMS_OLDER_THAN} -print)<commit_msg>Don't keep more builds than required.<commit_after>#!/bin/bash
# PARAMS:
# $1: Target ID (for example mb526).
# $2: "Name" of the ROM (this will be used as folder in the ROM_OUTPUT_DIRECTORY).
# PWD: This script has to be started from the "source code" folder.

BIN_DIR=${BIN_DIR:-"/home/android/"}
ROM_DATABASE_SCRIPT_DIR=${ROM_DATABASE_SCRIPT_DIR:-"${BIN_DIR}/cm-update-api/"}

DEVICE_ID=${DEVICE_ID:-"$1"}
ROM_SUBDIRECTORY=${ROM_SUBDIRECTORY:-"$2"}
LOCAL_MANIFEST=${LOCAL_MANIFEST:-"$3"}

if [[ -z "${DEVICE_ID}" ]]
then
	echo "Argument #1 (device ID) is required!"
	exit 1
fi

if [[ -z "${ROM_SUBDIRECTORY}" ]]
then
	echo "Argument #2 (ROM subdirectory) is required!"
	exit 1
fi

echo "Build-script was called for target '${DEVICE_ID}' and result directory '${ROM_SUBDIRECTORY}'."

PUBLIC_ROM_DIRECTORY=${PUBLIC_ROM_DIRECTORY:-"${BIN_DIR}/roms/${ROM_SUBDIRECTORY}"}
INCREMENTAL_UPDATES_DIRECTORY=${INCREMENTAL_UPDATES_DIRECTORY:-"${BIN_DIR}/incrementals/${ROM_SUBDIRECTORY}"}
TARGET_FILES_DIRECTORY=${TARGET_FILES_DIRECTORY:-"${BIN_DIR}/targetfiles/${ROM_SUBDIRECTORY}"}

ROM_OUTPUT_DIR="out/target/product/${DEVICE_ID}/"
TARGET_FILES_OUTPUT_DIR="${ROM_OUTPUT_DIR}/obj/PACKAGING/target_files_intermediates/"

DELETE_ROMS_OLDER_THAN=${DELETE_ROMS_OLDER_THAN:-7}

function disable_build {
	FILENAME="${1}"

	(cd $ROM_DATABASE_SCRIPT_DIR && \
		node disable-build.js \
			--device $DEVICE_ID \
			--filename $FILENAME \
			--subdirectory $ROM_SUBDIRECTORY \
			--disable_incrementals)
}

set -e
set -o xtrace
set -o pipefail

# Building will fail of no valid locale is set.
export LC_ALL=en_US.UTF-8
export LANG=en_US.UTF-8
export LANGUAGE=en_US.UTF-8

# Let's make sure JAVA_HOME is in our path and found first.
if [ ! -z "${JAVA_HOME}" ]
then
	echo "Using JDK from '${JAVA_HOME}'."
	export PATH="${JAVA_HOME}/bin:${PATH}"
fi

if [ -n "${LOCAL_MANIFEST}" ]
then
	echo "Using local manifest: ${LOCAL_MANIFEST}"

	if [ -e .repo/local_manifests/*.xml ]
	then
		rm .repo/local_manifests/*.xml
	fi

	SOURCE_MANIFEST_FILEPATH=$(readlink -e "./.repo/manifests/${LOCAL_MANIFEST}")
	ln -s "${SOURCE_MANIFEST_FILEPATH}" .repo/local_manifests/
fi

export USE_CCACHE=${USE_CCACHE:-1}

if [[ -z "${CCACHE_DIR}" ]]
then
	export CCACHE_DIR="${BIN_DIR}/.ccache/"
fi

if [[ "${SKIP_REPO_SYNC}" == "true" ]]
then
	echo "Skipping 'repo sync'"
else
	SOURCE_TIMESTAMP=$(date +"%s")

	echo "Starting 'repo sync'..."
	time repo sync
fi

export CM_BUILDTYPE=${CM_BUILDTYPE:-NIGHTLY}

echo "Getting CM prebuilts..."
time vendor/cm/get-prebuilts

echo "Setting up build environment..."
. ./build/envsetup.sh

IONICE="ionice -c3"
SCHEDULING="${IONICE} schedtool -B -n19 -e"

if [[ "${SKIP_MAKE_CLEAN}" == "true" ]]
then
	echo "Skipping make clean"

	# Remove potentially stale files.
	rm -f $ROM_OUTPUT_DIR/cm-*.zip
	rm -f $ROM_OUTPUT_DIR/incremental*.zip
	rm -f $ROM_OUTPUT_DIR/md5sum
	rm -f $TARGET_FILES_OUTPUT_DIR/cm_*.zip
	rm -f $ROM_OUTPUT_DIR/system/build.prop
else
	echo "Cleaning output directory..."
	time $SCHEDULING make -j1 clean
fi

# Required, because otherwise CM won't build.
set +e
set +o xtrace

echo "Configuring build..."
time breakfast "${DEVICE_ID}" || exit 1

echo "Starting build..."
time $SCHEDULING make -j1 bacon || exit 1

echo "Finished build!"

set -e
set -o xtrace

TARGET_ROM_ZIP=$ROM_OUTPUT_DIR/cm-*.zip
TARGET_ROM_MD5SUM=$TARGET_ROM_ZIP.md5sum
TARGET_FILES_ZIP=$TARGET_FILES_OUTPUT_DIR/cm_*target*.zip

# Don't use absolute paths in the md5sum file.
sed -r -i "s|$(readlink -e ${ROM_OUTPUT_DIR})/?||g" $TARGET_ROM_MD5SUM

ls -la ${PUBLIC_ROM_DIRECTORY}

echo "Removing builds older than ${DELETE_ROMS_OLDER_THAN} days..."

# Remove old builds.
# NOTE: -mtime only includes files older than N _full_ days - thus we subtract 1.
# See: http://unix.stackexchange.com/questions/92346/why-does-find-mtime-1-only-return-files-older-than-2-days
for FILE in $(find "${PUBLIC_ROM_DIRECTORY}" -type f -mtime +$((${DELETE_ROMS_OLDER_THAN} - 1)) -print)
{'fraction_non_alphanumeric': 0.15183109605108824, 'fraction_numerical': 0.008992571354098788, 'mean_word_length': 4.981293842556508, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1005785', 'n_tokens_mistral': 3046, 'n_tokens_neox': 2971, 'n_words': 687}