<commit_before>#!/bin/sh
# $Id$

# Here for a day when almost everything can be done from an unprivileged user.
# Unfortunately, that day is not today.
# So keep it for the places that would be critical if we had a choice.
priv () {
	if [ "${USE_SUDO}" = "YES" ] ; then
		sudo $*
	else
		$*
	fi
}

export FORFS="
"
export OLDFS=" 	
"
export SVNDATE="$(svn info --xml | awk '/<date>/, /<\/date>/' | sed 's@<date>@@g' | sed 's@</date>@@g' | sed 's@T@ @g' | cut -d '.' -f 1)"
export TRACKFILE_DATE="$(date -j -f "%Y-%m-%dT%H:%M:%S" "${SVNDATE}" "+%s")"

# BOOTDIR is the 'boot' directory; it's the root of the cd image
export BOOTDIR=${OBJDIR}/bdir

# FSDIR is the root of the root.fs image
export FSDIR=${OBJDIR}/fsdir

# The prefix for this version, for BOOTDIR, so we can avoid collisions with FreeBSD.
export BOOTPREFIX=/evoke/${VERSION}/${REVISION}

# PRODUCTDIR is where the filesystem images are stored.
export PRODUCTDIR=${BOOTPREFIX}/product

TARGETLIST="$(cat ${BUILDDIR}/targetlist | grep -v ^$ | grep -v ^#)"

for target in $(cat ${BUILDDIR}/targetlist | grep -v ^$ | grep -v ^#)
do
	# The TARGET and TARGET_ARCH are used by buildworld, and also internally
	# except for PC98, they are the same.
	export TARGET_HASH=$(echo ${target} | md5 -q)
	export TARGET=$(echo ${target} | cut -d ":" -f 3)
	export TARGET_ARCH="${TARGET}"

	# Release (eg, 6.3)
	export RELEASE=$(echo ${target} | cut -d ":" -f 4)

	# Kernel ABI (eg, 7 for 7.0-RELEASE or 7.1-RELEASE)
	export ABI=$(echo ${target} | cut -d ":" -f 2)

	# Kernel config definition; since boot loader versioning works now,
	# no need to use anything other then kernel
	export KERNCONF="kernel"

	# for installkernel/installworld and places in the script
	# that need to know where installworld is putting things
	export DESTDIR=${OBJDIR}/${TARGET_HASH}
	mkdir -p ${DESTDIR}

	# erm, why is this here? might be removable, older version probably treated 
	# them seperately.
	export WORKDIR=${DESTDIR}
	mkdir -p ${WORKDIR}

	# N_ prefix variables are for the file copies from DESTDIR to FSDIR, the 
	# targets are arch and abi specific.
	export N_DIR=/system/FreeBSD-${ABI}/${TARGET}
	export N_SHAREBIN=/system/share/bin
	export N_SHARELIB=/system/share/lib
	export N_BIN=${N_DIR}/bin
	export N_LIB=${N_DIR}/lib
	export N_LIBEXEC=${N_DIR}/libexec
	export N_BOOT=${N_DIR}/boot

	mkdir -p ${FSDIR}${N_BIN} 1>&2
	mkdir -p ${FSDIR}${N_LIB} 1>&2
	mkdir -p ${FSDIR}${N_LIBEXEC} 1>&2
	mkdir -p ${FSDIR}${N_BOOT} 1>&2

	# This forces same architecture buildworld/buildkernels to build cross-tools
	# and build-tools adequately.
	# ...
	#
	# DO NOT REMOVE UNDER PENALTY OF DEATH.
	export CROSS_BUILD_TESTING=yes

	# Extract source distfiles into ${DESTDIR}
#	SRCDIR="${NDISTDIR}/${RELEASE}" DISTS="src" ${ROOTDIR}/share/bin/distextract 1>&2
	export URL=$(echo ${target} | cut -d ":" -f 5)
	export URLREV=$(echo ${target} | cut -d ":" -f 6)
	export SRCDIR=${NDISTDIR}/${URL}/src
	mkdir -p ${SRCDIR}
	if [ -d ${SRCDIR}/.svn ] ; then
		cd ${SRCDIR}
		svn up -r ${URLREV} ${SRCDIR}
	else
		svn co http://svn.freebsd.org/base/${URL}@${URLREV} ${SRCDIR}
	fi
	cd ${SRCDIR} 
	mkdir -p ${DESTDIR}/usr/src
	tar -cpf - * | tar -xvpf - -C ${DESTDIR}/usr/src/
	ERROR="$?"
	if [ "${ERROR}" != "0" ] ; then
		# Wooo, real error message now.
		echo "Source dist extraction failed."
		echo "Error code: ${ERROR}"
		exit 1
	fi

	echo -n " * ${target} = Patching World"


	# Before we do anything else, apply all diffs in the shared directory for the release.

	PATCHLIST="$(echo ${BUILDDIR}/targets/FreeBSD/${RELEASE}/share/patches/*)"
	if [ "${PATCHLIST}" != "" ] ; then
		cd ${WORKDIR}/usr/src

		for patchfile in ${PATCHLIST}
		do
			patch -p0 <"${patchfile}" >&2
		done
	fi

	# Ok, we need to patch around some hard coded paths in src/
	cd ${WORKDIR}/usr/src/sys/boot/

	# This changes per ${target}
	export BOOTPATH="${BOOTPREFIX}/FreeBSD/${RELEASE}/${TARGET}"
	mkdir -p ${BOOTDIR}${BOOTPATH}

	# Get rid of /libexec/ld-elf.so.1, and use the rtld in each arch+abi specific directory.
	for file in /usr/src/sys/${TARGET}/${TARGET}/elf_machdep.c
	do
		sed -i .bak "s@/libexec/ld-elf.so.1@${N_LIBEXEC}/ld-elf.so.1" ${WORKDIR}${file} 1>&2
	done

	# Point rtld to ${N_LIBDIR} instead of the /lib, hopefully, this will allow us to compile the system with 
	# only init statically linked. 
	sed -i .bak "s@/lib:/usr/lib@${N_LIBDIR}" ${WORKDIR}/usr/src/libexec/rtld-elf/rtld.h 1>&2

	# Remove the ld-elf32.so.1 kludge in favor of a more universal method.
	for file in /usr/src/sys/compat/ia32/ia32_sysvec.c /usr/src/libexec/rtld-elf/rtld.c /usr/src/libexec/rtld-elf/debug.h
	do
		sed -i .bak "s@/libexec/ld-elf32.so.1@/system/FreeBSD-${ABI}/i386/libexec/ld-elf.so.1" ${WORKDIR}${file} 1>&2
	done

	# Patch these files to our paths, so they don't collide.
	# This is the bulk of the boot loader versioning support.
	sed -i .bak "s@/boot/device.hints@${BOOTPATH}/device.hints@g" ${WORKDIR}/usr/src/sys/boot/forth/loader.conf 1>&2
	sed -i .bak "s@/boot/loader.conf.local@/evoke/site.conf@g" ${WORKDIR}/usr/src/sys/boot/forth/loader.conf 1>&2
	sed -i .bak "s@/boot/loader.conf@${BOOTPREFIX}/loader.conf@g" ${WORKDIR}/usr/src/sys/boot/forth/loader.conf 1>&2
	for file in $(cat ${BUILDDIR}/bootlist)
	do
	    # This works for most.
	    sed -i .bak "s@/boot/@${BOOTPATH}/@g" ${WORKDIR}${file} 1>&2
	    # This is for cdboot. Case specific
	    sed -i .bak "s@/BOOT/@$(echo ${BOOTPATH} | tr a-z A-Z)/@g" ${WORKDIR}${file} 1>&2
	done

	# Get rid of this latency addition in pxeboot (fixed in 7.0, but necessary on 6.x)
	sed -i .bak '/pxe_setnfshandle(rootpath);/d' ${WORKDIR}/usr/src/sys/boot/i386/libi386/pxe.c 1>&2

	# rescue binaries hard code /rescue; for us, it's more productive to point it to the abi/arch 
	# directory directly. It makes systart transparent.
	sed -i .bak "s@\"/rescue@\"${N_BIN}@g" ${WORKDIR}/usr/src/include/paths.h 1>&2


	# we don't have a real /etc, so point init to systart.
	sed -i .bak "s_\"/etc/rc_\"${N_SHAREBIN}/systart_g" ${WORKDIR}/usr/src/sbin/init/pathnames.h 1>&2

	# Patch for lib/libmagic's build-tools target.
	sed -i .bak "s_usr/sbin usr/share/misc_usr/sbin usr/share/misc config_g" ${WORKDIR}/usr/src/Makefile.inc1 1>&2

	echo ""
	# We use TMPDIR so that it can be different then the OBJDIR, as TMPDIR is write heavy.
	export MAKEOBJDIRPREFIX=${OBJDIR}/obj/${TARGET_HASH}
	mkdir -p ${MAKEOBJDIRPREFIX}


	# This copies nsrc to src/, to compile our own code. For some reason does not work, but non-critical.

	cd ${BUILDDIR}
	tar -cf - nsrc | tar -xf - -C ${DESTDIR}/usr/src/

	# Allow us to build both statically and dynamically. Note, it needs to be automated more.

	case "${1}" in
		[sS][tT][aA][tT][iI][cC])
			. ${BUILDDIR}/static.sh
		;;
		[dD][yY][nN][aA][mM][iIk][cC])
			. ${BUILDDIR}/dynamic.sh
		;;
		*)
			. ${BUILDDIR}/dynamic.sh
		;;
	esac

	echo -n " * ${target} = Compressing Kernel"
	cd ${DESTDIR}/boot/${KERNCONF}/

	# Grab port modules and copy them to the kernel directory.
	for file in $(grep ^M ${BUILDDIR}/portlist | cut -d : -f 2)
	do
		cp ${DESTDIR}/usr/local/modules/${file}.ko ./
	done

#	gzip -9 kernel
	for file in $(grep ^M ${BUILDDIR}/portlist | cut -d : -f 2) ${MODULES}
	do
#		gzip -9 ${file}.ko
		mv ${file}.ko ${file}.ka
	done
	# Remove everything that is not gzipped.<commit_msg>* Add logic to support modules stored in /boot/modules in addition to /usr/local/modules.<commit_after>#!/bin/sh
# $Id$

# Here for a day when almost everything can be done from an unprivileged user.
# Unfortunately, that day is not today.
# So keep it for the places that would be critical if we had a choice.
priv () {
	if [ "${USE_SUDO}" = "YES" ] ; then
		sudo $*
	else
		$*
	fi
}

export FORFS="
"
export OLDFS=" 	
"
export SVNDATE="$(svn info --xml | awk '/<date>/, /<\/date>/' | sed 's@<date>@@g' | sed 's@</date>@@g' | sed 's@T@ @g' | cut -d '.' -f 1)"
export TRACKFILE_DATE="$(date -j -f "%Y-%m-%dT%H:%M:%S" "${SVNDATE}" "+%s")"

# BOOTDIR is the 'boot' directory; it's the root of the cd image
export BOOTDIR=${OBJDIR}/bdir

# FSDIR is the root of the root.fs image
export FSDIR=${OBJDIR}/fsdir

# The prefix for this version, for BOOTDIR, so we can avoid collisions with FreeBSD.
export BOOTPREFIX=/evoke/${VERSION}/${REVISION}

# PRODUCTDIR is where the filesystem images are stored.
export PRODUCTDIR=${BOOTPREFIX}/product

TARGETLIST="$(cat ${BUILDDIR}/targetlist | grep -v ^$ | grep -v ^#)"

for target in $(cat ${BUILDDIR}/targetlist | grep -v ^$ | grep -v ^#)
do
	# The TARGET and TARGET_ARCH are used by buildworld, and also internally
	# except for PC98, they are the same.
	export TARGET_HASH=$(echo ${target} | md5 -q)
	export TARGET=$(echo ${target} | cut -d ":" -f 3)
	export TARGET_ARCH="${TARGET}"

	# Release (eg, 6.3)
	export RELEASE=$(echo ${target} | cut -d ":" -f 4)

	# Kernel ABI (eg, 7 for 7.0-RELEASE or 7.1-RELEASE)
	export ABI=$(echo ${target} | cut -d ":" -f 2)

	# Kernel config definition; since boot loader versioning works now,
	# no need to use anything other then kernel
	export KERNCONF="kernel"

	# for installkernel/installworld and places in the script
	# that need to know where installworld is putting things
	export DESTDIR=${OBJDIR}/${TARGET_HASH}
	mkdir -p ${DESTDIR}

	# erm, why is this here? might be removable, older version probably treated 
	# them seperately.
	export WORKDIR=${DESTDIR}
	mkdir -p ${WORKDIR}

	# N_ prefix variables are for the file copies from DESTDIR to FSDIR, the 
	# targets are arch and abi specific.
	export N_DIR=/system/FreeBSD-${ABI}/${TARGET}
	export N_SHAREBIN=/system/share/bin
	export N_SHARELIB=/system/share/lib
	export N_BIN=${N_DIR}/bin
	export N_LIB=${N_DIR}/lib
	export N_LIBEXEC=${N_DIR}/libexec
	export N_BOOT=${N_DIR}/boot

	mkdir -p ${FSDIR}${N_BIN} 1>&2
	mkdir -p ${FSDIR}${N_LIB} 1>&2
	mkdir -p ${FSDIR}${N_LIBEXEC} 1>&2
	mkdir -p ${FSDIR}${N_BOOT} 1>&2

	# This forces same architecture buildworld/buildkernels to build cross-tools
	# and build-tools adequately.
	# ...
	#
	# DO NOT REMOVE UNDER PENALTY OF DEATH.
	export CROSS_BUILD_TESTING=yes

	# Extract source distfiles into ${DESTDIR}
#	SRCDIR="${NDISTDIR}/${RELEASE}" DISTS="src" ${ROOTDIR}/share/bin/distextract 1>&2
	export URL=$(echo ${target} | cut -d ":" -f 5)
	export URLREV=$(echo ${target} | cut -d ":" -f 6)
	export SRCDIR=${NDISTDIR}/${URL}/src
	mkdir -p ${SRCDIR}
	if [ -d ${SRCDIR}/.svn ] ; then
		cd ${SRCDIR}
		svn up -r ${URLREV} ${SRCDIR}
	else
		svn co http://svn.freebsd.org/base/${URL}@${URLREV} ${SRCDIR}
	fi
	cd ${SRCDIR} 
	mkdir -p ${DESTDIR}/usr/src
	tar -cpf - * | tar -xvpf - -C ${DESTDIR}/usr/src/
	ERROR="$?"
	if [ "${ERROR}" != "0" ] ; then
		# Wooo, real error message now.
		echo "Source dist extraction failed."
		echo "Error code: ${ERROR}"
		exit 1
	fi

	echo -n " * ${target} = Patching World"


	# Before we do anything else, apply all diffs in the shared directory for the release.

	PATCHLIST="$(echo ${BUILDDIR}/targets/FreeBSD/${RELEASE}/share/patches/*)"
	if [ "${PATCHLIST}" != "" ] ; then
		cd ${WORKDIR}/usr/src

		for patchfile in ${PATCHLIST}
		do
			patch -p0 <"${patchfile}" >&2
		done
	fi

	# Ok, we need to patch around some hard coded paths in src/
	cd ${WORKDIR}/usr/src/sys/boot/

	# This changes per ${target}
	export BOOTPATH="${BOOTPREFIX}/FreeBSD/${RELEASE}/${TARGET}"
	mkdir -p ${BOOTDIR}${BOOTPATH}

	# Get rid of /libexec/ld-elf.so.1, and use the rtld in each arch+abi specific directory.
	for file in /usr/src/sys/${TARGET}/${TARGET}/elf_machdep.c
	do
		sed -i .bak "s@/libexec/ld-elf.so.1@${N_LIBEXEC}/ld-elf.so.1" ${WORKDIR}${file} 1>&2
	done

	# Point rtld to ${N_LIBDIR} instead of the /lib, hopefully, this will allow us to compile the system with 
	# only init statically linked. 
	sed -i .bak "s@/lib:/usr/lib@${N_LIBDIR}" ${WORKDIR}/usr/src/libexec/rtld-elf/rtld.h 1>&2

	# Remove the ld-elf32.so.1 kludge in favor of a more universal method.
	for file in /usr/src/sys/compat/ia32/ia32_sysvec.c /usr/src/libexec/rtld-elf/rtld.c /usr/src/libexec/rtld-elf/debug.h
	do
		sed -i .bak "s@/libexec/ld-elf32.so.1@/system/FreeBSD-${ABI}/i386/libexec/ld-elf.so.1" ${WORKDIR}${file} 1>&2
	done

	# Patch these files to our paths, so they don't collide.
	# This is the bulk of the boot loader versioning support.
	sed -i .bak "s@/boot/device.hints@${BOOTPATH}/device.hints@g" ${WORKDIR}/usr/src/sys/boot/forth/loader.conf 1>&2
	sed -i .bak "s@/boot/loader.conf.local@/evoke/site.conf@g" ${WORKDIR}/usr/src/sys/boot/forth/loader.conf 1>&2
	sed -i .bak "s@/boot/loader.conf@${BOOTPREFIX}/loader.conf@g" ${WORKDIR}/usr/src/sys/boot/forth/loader.conf 1>&2
	for file in $(cat ${BUILDDIR}/bootlist)
	do
	    # This works for most.
	    sed -i .bak "s@/boot/@${BOOTPATH}/@g" ${WORKDIR}${file} 1>&2
	    # This is for cdboot. Case specific
	    sed -i .bak "s@/BOOT/@$(echo ${BOOTPATH} | tr a-z A-Z)/@g" ${WORKDIR}${file} 1>&2
	done

	# Get rid of this latency addition in pxeboot (fixed in 7.0, but necessary on 6.x)
	sed -i .bak '/pxe_setnfshandle(rootpath);/d' ${WORKDIR}/usr/src/sys/boot/i386/libi386/pxe.c 1>&2

	# rescue binaries hard code /rescue; for us, it's more productive to point it to the abi/arch 
	# directory directly. It makes systart transparent.
	sed -i .bak "s@\"/rescue@\"${N_BIN}@g" ${WORKDIR}/usr/src/include/paths.h 1>&2


	# we don't have a real /etc, so point init to systart.
	sed -i .bak "s_\"/etc/rc_\"${N_SHAREBIN}/systart_g" ${WORKDIR}/usr/src/sbin/init/pathnames.h 1>&2

	# Patch for lib/libmagic's build-tools target.
	sed -i .bak "s_usr/sbin usr/share/misc_usr/sbin usr/share/misc config_g" ${WORKDIR}/usr/src/Makefile.inc1 1>&2

	echo ""
	# We use TMPDIR so that it can be different then the OBJDIR, as TMPDIR is write heavy.
	export MAKEOBJDIRPREFIX=${OBJDIR}/obj/${TARGET_HASH}
	mkdir -p ${MAKEOBJDIRPREFIX}


	# This copies nsrc to src/, to compile our own code. For some reason does not work, but non-critical.

	cd ${BUILDDIR}
	tar -cf - nsrc | tar -xf - -C ${DESTDIR}/usr/src/

	# Allow us to build both statically and dynamically. Note, it needs to be automated more.

	case "${1}" in
		[sS][tT][aA][tT][iI][cC])
			. ${BUILDDIR}/static.sh
		;;
		[dD][yY][nN][aA][mM][iIk][cC])
			. ${BUILDDIR}/dynamic.sh
		;;
		*)
			. ${BUILDDIR}/dynamic.sh
		;;
	esac

	echo -n " * ${target} = Compressing Kernel"
	cd ${DESTDIR}/boot/${KERNCONF}/

	# Grab port modules and copy them to the kernel directory.
	for file in $(grep ^M ${BUILDDIR}/portlist | cut -d : -f 2)
	do
		if [ -f ${DESTDIR}/usr/local/modules/${file}.ko ] ; then
			cp ${DESTDIR}/usr/local/modules/${file}.ko ./
		else
			cp ${DESTDIR}/boot/modules/${file}.ko ./
		fi
	done

#	gzip -9 kernel
	for file in $(grep ^M ${BUILDDIR}/portlist | cut -d : -f 2) ${MODULES}
	do
#		gzip -9 ${file}.ko
		mv ${file}.ko ${file}.ka
	done
	# Remove everything that is not gzipped.
{'fraction_non_alphanumeric': 0.1680227827502034, 'fraction_numerical': 0.011662598318416057, 'mean_word_length': 3.5904139433551197, 'pattern_counts': {'":': 10, '<': 13, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '458199', 'n_tokens_mistral': 6031, 'n_tokens_neox': 5690, 'n_words': 1671}