<commit_before>#!/bin/bash

# setup.sh
# Setup or update mirrors

set -e -o pipefail

PROGRAM=$(readlink -f "$0")
PROGRAM_NAME="$(basename "$0")"
PROGRAM_DIR="$(dirname "$0")"

. "${PROGRAM_DIR}"/common.sh
no_package=true
init "$@"

function checkOrLinkDebootstrapScript()
{
	if [ ! -f /usr/share/debootstrap/scripts/$1 ]; then
		if [[ "$UBUNTU_DISTROS $UBUNTU_OLDDISTROS" == "*$1*" ]]; then
			basedistro=gutsy
		else
			basedistro=sid
		fi
		sudo ln -s /usr/share/debootstrap/scripts/$basedistro /usr/share/debootstrap/scripts/$1
	fi
}

function addmirror()
{
	OTHERMIRROR="$OTHERMIRROR${OTHERMIRROR:+|}$1"
}

# Create a source.list template file that will be used in the chroot
function createSources()
{
	cat > $1 <<EOF
$(echo $OTHERMIRROR | tr '|' '\n')
EOF
}

function enableBackports()
{
	cat > $1 <<EOF
Package: *
Pin: release a=$2-backports
Pin-Priority: 500
EOF
}

function checkAndInstallRequirements()
{
	local TOINSTALL GROUP
	if [ ! -x /usr/bin/mk-sbuild ]; then
		TOINSTALL="$TOINSTALL ubuntu-dev-tools"
	fi
	if [ ! -x /usr/bin/sbuild ]; then
		TOINSTALL="$TOINSTALL sbuild"
		touch $HOME/.sbuildrc
	fi

	if [ -n "$TOINSTALL" ]; then
		log "Installing prerequisites: $TOINSTALL"
		sudo apt-get update
		sudo apt-get -qy install $TOINSTALL
	fi

	GROUP=$(groups | grep sbuild || true)
	if [ -z "$GROUP" ]; then
		sudo adduser $USER sbuild > /dev/null || true
		local TEMPFILE=$(mktemp)
		local ARGS=()
		while (( $# )); do
			ARGS+=("\"$1\"")
			shift
		done
		echo "${ARGS[@]}" > $TEMPFILE
		chmod +x $TEMPFILE

		log "Calling $(basename ${ARGS[0]}) recursively"
		sg sbuild $TEMPFILE
		rm $TEMPFILE
		exit
	fi

	# We have to install a current version of mk-sbuild because trying to build newer dists
	# on an older dist might have different requirements than the system provided version
	# of mk-sbuild provides (e.g. on xenial when trying to build a bionic chroot).
	if [ ! -f $HOME/bin/mk-sbuild -o ! -f $HOME/bin/mk-sbuild.v${MKSBUILD_VERSION} ]; then
		log "Getting version ${MKSBUILD_VERSION} of mk-sbuild"
		mkdir -p $HOME/bin
		rm -f $HOME/bin/mk-sbuild*
		TRACE wget --output-document=$HOME/bin/mk-sbuild https://git.launchpad.net/ubuntu-dev-tools/plain/mk-sbuild?h=${MKSBUILD_VERSION}
		chmod +x $HOME/bin/mk-sbuild
		touch $HOME/bin/mk-sbuild.v${MKSBUILD_VERSION}
	fi

	installFileSbuildrc
}

function installFileSbuildrc()
{
	SBUILDRC_PATH="${HOME}/.sbuildrc"
	if diff "${SBUILDRC_PATH}" "${PROGRAM_DIR}"/sbuildrc; then
		return
	fi
	log "Installing .sbuildrc"
	# Temporarily backup any existing .sbuildrc, in case we are overwriting something good.
	cp -v "${SBUILDRC_PATH}" $(mktemp /tmp/old-sbuildrc.XXXXXXXXXX) || true
	cp -v "${PROGRAM_DIR}"/sbuildrc "${SBUILDRC_PATH}"
}

function copyInKeyrings()
{
	[ -f "${KEYRINGLLSO}" ] && sudo cp "${KEYRINGLLSO}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRINGPSO}" ] && sudo cp "${KEYRINGPSO}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRINGNODE}" ] && sudo cp "${KEYRINGNODE}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRINGMICROSOFT}" ] && sudo cp "${KEYRINGMICROSOFT}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRING_MONO}" ] && sudo cp "${KEYRING_MONO}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
}

function addExtraRepositories()
{
	TMPFILE="$(mktemp)"
	createSources "${TMPFILE}"
	sudo cp "${TMPFILE}" "${SCHROOTDIR}/${D}-${A}/etc/apt/sources.list.d/extra.list"
	rm "${TMPFILE}"
}

WORKDIR="${WORKSPACE:-$(realpath "${PROGRAM_DIR}")}"

cd "${WORKDIR}"

checkAndInstallRequirements $PROGRAM "$@"

KEYRINGLLSO="$WORKDIR/llso-keyring-2013.gpg"
KEYRINGPSO="$WORKDIR/pso-keyring-2016.gpg"
KEYRINGNODE="$WORKDIR/nodesource-keyring.gpg"
KEYRINGMICROSOFT="$WORKDIR/microsoft.asc.gpg"
KEYRING_MONO="$WORKDIR/mono-project.asc.gpg"

if [ ! -f ${KEYRINGPSO} ]; then
	wget --output-document=${KEYRINGPSO} https://packages.sil.org/keys/pso-keyring-2016.gpg
fi

if [ ! -f ${KEYRINGLLSO} ]; then
	wget --output-document=${KEYRINGLLSO} http://linux.lsdev.sil.org/keys/llso-keyring-2013.gpg
fi

if [ ! -f ${KEYRINGNODE} ]; then
	# Download node key and convert to keyring.

	NODE_KEY="$(mktemp -d)/nodesource-key.asc"
	# Use a temporary, intermediate keyring since it may be keybox format on Ubuntu 20.04, and we need it to be an older format for apt.
	TMP_KEYRING=$(mktemp)
	wget --output-document=${NODE_KEY} https://deb.nodesource.com/gpgkey/nodesource.gpg.key<commit_msg>[Packaging] Improve robustness of setup.sh script<commit_after>#!/bin/bash

# setup.sh
# Setup or update mirrors

set -e -o pipefail

PROGRAM=$(readlink -f "$0")
PROGRAM_NAME="$(basename "$0")"
PROGRAM_DIR="$(realpath $(dirname "$0"))"

. "${PROGRAM_DIR}"/common.sh
no_package=true
init "$@"

function checkOrLinkDebootstrapScript()
{
	if [ ! -f /usr/share/debootstrap/scripts/$1 ]; then
		if [[ "$UBUNTU_DISTROS $UBUNTU_OLDDISTROS" == "*$1*" ]]; then
			basedistro=gutsy
		else
			basedistro=sid
		fi
		sudo ln -s /usr/share/debootstrap/scripts/$basedistro /usr/share/debootstrap/scripts/$1
	fi
}

function addmirror()
{
	OTHERMIRROR="$OTHERMIRROR${OTHERMIRROR:+|}$1"
}

# Create a source.list template file that will be used in the chroot
function createSources()
{
	cat > $1 <<EOF
$(echo $OTHERMIRROR | tr '|' '\n')
EOF
}

function enableBackports()
{
	cat > $1 <<EOF
Package: *
Pin: release a=$2-backports
Pin-Priority: 500
EOF
}

function checkAndInstallRequirements()
{
	local TOINSTALL GROUP
	if [ ! -x /usr/bin/mk-sbuild ]; then
		TOINSTALL="$TOINSTALL ubuntu-dev-tools"
	fi
	if [ ! -x /usr/bin/sbuild ]; then
		TOINSTALL="$TOINSTALL sbuild"
		touch $HOME/.sbuildrc
	elif [ ! -f $HOME/.sbuildrc ]; then
		touch $HOME/.sbuildrc
	fi

	if [ -n "$TOINSTALL" ]; then
		log "Installing prerequisites: $TOINSTALL"
		sudo apt-get update
		sudo apt-get -qy install $TOINSTALL
	fi

	GROUP=$(groups | grep sbuild || true)
	if [ -z "$GROUP" ]; then
		sudo adduser $USER sbuild > /dev/null || true
		local TEMPFILE=$(mktemp)
		local ARGS=()
		while (( $# )); do
			ARGS+=("\"$1\"")
			shift
		done
		echo "${ARGS[@]}" > $TEMPFILE
		chmod +x $TEMPFILE

		log "Calling $(basename ${ARGS[0]}) recursively"
		sg sbuild $TEMPFILE
		rm $TEMPFILE
		exit
	fi

	# We have to install a current version of mk-sbuild because trying to build newer dists
	# on an older dist might have different requirements than the system provided version
	# of mk-sbuild provides (e.g. on xenial when trying to build a bionic chroot).
	if [ ! -f $HOME/bin/mk-sbuild -o ! -f $HOME/bin/mk-sbuild.v${MKSBUILD_VERSION} ]; then
		log "Getting version ${MKSBUILD_VERSION} of mk-sbuild"
		mkdir -p $HOME/bin
		rm -f $HOME/bin/mk-sbuild*
		TRACE wget --output-document=$HOME/bin/mk-sbuild https://git.launchpad.net/ubuntu-dev-tools/plain/mk-sbuild?h=${MKSBUILD_VERSION}
		chmod +x $HOME/bin/mk-sbuild
		touch $HOME/bin/mk-sbuild.v${MKSBUILD_VERSION}
	fi

	installFileSbuildrc
}

function installFileSbuildrc()
{
	SBUILDRC_PATH="${HOME}/.sbuildrc"
	if diff "${SBUILDRC_PATH}" "${PROGRAM_DIR}"/sbuildrc; then
		return
	fi
	log "Installing .sbuildrc"
	# Temporarily backup any existing .sbuildrc, in case we are overwriting something good.
	cp -v "${SBUILDRC_PATH}" $(mktemp /tmp/old-sbuildrc.XXXXXXXXXX) || true
	cp -v "${PROGRAM_DIR}"/sbuildrc "${SBUILDRC_PATH}"
}

function copyInKeyrings()
{
	[ -f "${KEYRINGLLSO}" ] && sudo cp "${KEYRINGLLSO}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRINGPSO}" ] && sudo cp "${KEYRINGPSO}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRINGNODE}" ] && sudo cp "${KEYRINGNODE}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRINGMICROSOFT}" ] && sudo cp "${KEYRINGMICROSOFT}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
	[ -f "${KEYRING_MONO}" ] && sudo cp "${KEYRING_MONO}" "${SCHROOTDIR}/${D}-${A}/etc/apt/trusted.gpg.d/"
}

function addExtraRepositories()
{
	TMPFILE="$(mktemp)"
	createSources "${TMPFILE}"
	sudo cp "${TMPFILE}" "${SCHROOTDIR}/${D}-${A}/etc/apt/sources.list.d/extra.list"
	rm "${TMPFILE}"
}

WORKDIR="${WORKSPACE:-${PROGRAM_DIR}}"

cd "${WORKDIR}"

checkAndInstallRequirements $PROGRAM "$@"

KEYRINGLLSO="$WORKDIR/llso-keyring-2013.gpg"
KEYRINGPSO="$WORKDIR/pso-keyring-2016.gpg"
KEYRINGNODE="$WORKDIR/nodesource-keyring.gpg"
KEYRINGMICROSOFT="$WORKDIR/microsoft.asc.gpg"
KEYRING_MONO="$WORKDIR/mono-project.asc.gpg"

if [ ! -f ${KEYRINGPSO} ]; then
	wget --output-document=${KEYRINGPSO} https://packages.sil.org/keys/pso-keyring-2016.gpg
fi

if [ ! -f ${KEYRINGLLSO} ]; then
	wget --output-document=${KEYRINGLLSO} http://linux.lsdev.sil.org/keys/llso-keyring-2013.gpg
fi

if [ ! -f ${KEYRINGNODE} ]; then
	# Download node key and convert to keyring.

	NODE_KEY="$(mktemp -d)/nodesource-key.asc"
	# Use a temporary, intermediate keyring since it may be keybox format on Ubuntu 20.04, and we need it to be an older format for apt.
	TMP_KEYRING=$(mktemp)
	wget --output-document=${NODE_KEY} https://deb.nodesource.com/gpgkey/nodesource.gpg.key
{'fraction_non_alphanumeric': 0.17747899159663866, 'fraction_numerical': 0.00784313725490196, 'mean_word_length': 4.325775656324582, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1591702', 'n_tokens_mistral': 3708, 'n_tokens_neox': 3458, 'n_words': 760}