<commit_before>#!/usr/bin/env bash

BUILDAH_BINARY=${BUILDAH_BINARY:-$(dirname ${BASH_SOURCE})/../bin/buildah}
IMGTYPE_BINARY=${IMGTYPE_BINARY:-$(dirname ${BASH_SOURCE})/../bin/imgtype}
COPY_BINARY=${COPY_BINARY:-$(dirname ${BASH_SOURCE})/../bin/copy}
TESTSDIR=${TESTSDIR:-$(dirname ${BASH_SOURCE})}
STORAGE_DRIVER=${STORAGE_DRIVER:-vfs}
PATH=$(dirname ${BASH_SOURCE})/../bin:${PATH}
OCI=$(${BUILDAH_BINARY} info --format '{{.host.OCIRuntime}}' || command -v runc || command -v crun)
# Default timeout for a buildah command.
BUILDAH_TIMEOUT=${BUILDAH_TIMEOUT:-300}

# We don't invoke gnupg directly in many places, but this avoids ENOTTY errors
# when we invoke it directly in batch mode, and CI runs us without a terminal
# attached.
export GPG_TTY=/dev/null

function setup(){
    setup_tests
}

function setup_tests() {
    pushd "$(dirname "$(readlink -f "$BASH_SOURCE")")"

    # buildah/podman: "repository name must be lowercase".
    # me: "but it's a local file path, not a repository name!"
    # buildah/podman: "i dont care. no caps anywhere!"
    TESTDIR=$(mktemp -d --dry-run --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} buildah_tests.XXXXXX | tr A-Z a-z)
    mkdir --mode=0700 $TESTDIR

    mkdir -p ${TESTDIR}/{root,runroot,sigstore,registries.d}
    cat >${TESTDIR}/registries.d/default.yaml <<EOF
default-docker:
  sigstore-staging: file://${TESTDIR}/sigstore
docker:
  registry.access.redhat.com:
    sigstore: https://access.redhat.com/webassets/docker/content/sigstore
  registry.redhat.io:
    sigstore: https://registry.redhat.io/containers/sigstore
EOF

    # Common options for all buildah and podman invocations
    ROOTDIR_OPTS="--root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER}"
    BUILDAH_REGISTRY_OPTS="--registries-conf ${TESTSDIR}/registries.conf --registries-conf-dir ${TESTDIR}/registries.d --short-name-alias-conf ${TESTDIR}/cache/shortnames.conf"
    PODMAN_REGISTRY_OPTS="--registries-conf ${TESTSDIR}/registries.conf"
}

function starthttpd() {
    pushd ${2:-${TESTDIR}} > /dev/null
    go build -o serve ${TESTSDIR}/serve/serve.go
    portfile=$(mktemp)
    if test -z "${portfile}"; then
        echo error creating temporaty file
        exit 1
    fi
    ./serve ${1:-${BATS_TMPDIR}} 0 ${portfile} &
    HTTP_SERVER_PID=$!
    waited=0
    while ! test -s ${portfile} ; do
        sleep 0.1
        if test $((++waited)) -ge 300 ; then
            echo test http server did not start within timeout
            exit 1
        fi
    done
    HTTP_SERVER_PORT=$(cat ${portfile})
    rm -f ${portfile}
    popd > /dev/null
}

function stophttpd() {
    if test -n "$HTTP_SERVER_PID" ; then
        kill -HUP ${HTTP_SERVER_PID}
        unset HTTP_SERVER_PID
        unset HTTP_SERVER_PORT
    fi
    true
}

function teardown(){
    teardown_tests
}

function teardown_tests() {
    stophttpd
    stop_git_daemon

    # Workaround for #1991 - buildah + overlayfs leaks mount points.
    # Many tests leave behind /var/tmp/.../root/overlay and sub-mounts;
    # let's find those and clean them up, otherwise 'rm -rf' fails.
    # 'sort -r' guarantees that we umount deepest subpaths first.
    mount |\
        awk '$3 ~ testdir { print $3 }' testdir="^${TESTDIR}/" |\
        sort -r |\
        xargs --no-run-if-empty --max-lines=1 umount

    rm -fr ${TESTDIR}

    popd
}

function normalize_image_name() {
    for img in "$@"; do
        if [[ "${img##*/}" == "$img" ]] ; then
            echo -n docker.io/library/"$img"
        elif [[ docker.io/"${img##*/}" == "$img" ]] ; then
            echo -n docker.io/library/"${img##*/}"
        else
            echo -n "$img"
        fi
    done
}

function _prefetch() {
    if [ -z "${_BUILDAH_IMAGE_CACHEDIR}" ]; then
        _pgid=$(sed -ne 's/^NSpgid:\s*//p' /proc/$$/status)
        export _BUILDAH_IMAGE_CACHEDIR=${BATS_TMPDIR}/buildah-image-cache.$_pgid
        mkdir -p ${_BUILDAH_IMAGE_CACHEDIR}
    fi

    for img in "$@"; do
        img=$(normalize_image_name "$img")
        echo "# [checking for: $img]" >&2
        fname=$(tr -c a-zA-Z0-9.- - <<< "$img")
        if [ -d $_BUILDAH_IMAGE_CACHEDIR/$fname ]; then
            echo "# [restoring from cache: $_BUILDAH_IMAGE_CACHEDIR / $img]" >&2
            copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$img"
        else
            rm -fr $_BUILDAH_IMAGE_CACHEDIR/$fname
            echo "# [copy docker://$img dir:$_BUILDAH_IMAGE_CACHEDIR/$fname]" >&2
            for attempt in $(seq 3) ; do
                if copy docker://"$img" dir:$_BUILDAH_IMAGE_CACHEDIR/$fname ; then
                    break
                fi
                sleep 5
            done
            echo "# [copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:$img]" >&2
            copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$img"
        fi
    done
}

function createrandom() {
    dd if=/dev/urandom bs=1 count=${2:-256} of=${1:-${BATS_TMPDIR}/randomfile} status=none
}

function buildah() {
    ${BUILDAH_BINARY} ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
}

function imgtype() {
    ${IMGTYPE_BINARY} ${ROOTDIR_OPTS} "$@"
}

function copy() {
    ${COPY_BINARY} --max-parallel-downloads=1 ${ROOTDIR_OPTS} ${BUILDAH_REGISTRY_OPTS} "$@"
}

function podman() {
    command podman ${PODMAN_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
}

#################
#  run_buildah  #  Invoke buildah, with timeout, using BATS 'run'
#################
#
# This is the preferred mechanism for invoking buildah:
#
#  * we use 'timeout' to abort (with a diagnostic) if something
#    takes too long; this is preferable to a CI hang.
#  * we log the command run and its output. This doesn't normally
#    appear in BATS output, but it will if there's an error.
#  * we check exit status. Since the normal desired code is 0,
#    that's the default; but the first argument can override:
#
#     run_buildah 125  nonexistent-subcommand
#     run_buildah '?'  some-other-command       # let our caller check status
#
# Since we use the BATS 'run' mechanism, $output and $status will be
# defined for our caller.
#
function run_buildah() {
    # Number as first argument = expected exit code; default 0
    # --retry as first argument = retry 3 times on error (eg registry flakes)
    local expected_rc=0
    local retry=1
    case "$1" in
        [0-9])           expected_rc=$1; shift;;
        [1-9][0-9])      expected_rc=$1; shift;;
        [12][0-9][0-9])  expected_rc=$1; shift;;
        '?')             expected_rc=  ; shift;;  # ignore exit code
        --retry)         retry=3;        shift;;  # retry network flakes
    esac

    # Remember command args, for possible use in later diagnostic messages
    MOST_RECENT_BUILDAH_COMMAND="buildah $*"

    while [ $retry -gt 0 ]; do
        retry=$(( retry - 1 ))

        # stdout is only emitted upon error; this echo is to help a debugger
        echo "\$ $BUILDAH_BINARY $*"
        run env CONTAINERS_CONF=${CONTAINERS_CONF:-$(dirname ${BASH_SOURCE})/containers.conf} timeout --foreground --kill=10 $BUILDAH_TIMEOUT ${BUILDAH_BINARY} ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
        # without "quotes", multiple lines are glommed together into one
        if [ -n "$output" ]; then
            echo "$output"<commit_msg>test: unshare mount/umount if test is_rootless<commit_after>#!/usr/bin/env bash

BUILDAH_BINARY=${BUILDAH_BINARY:-$(dirname ${BASH_SOURCE})/../bin/buildah}
IMGTYPE_BINARY=${IMGTYPE_BINARY:-$(dirname ${BASH_SOURCE})/../bin/imgtype}
COPY_BINARY=${COPY_BINARY:-$(dirname ${BASH_SOURCE})/../bin/copy}
TESTSDIR=${TESTSDIR:-$(dirname ${BASH_SOURCE})}
STORAGE_DRIVER=${STORAGE_DRIVER:-vfs}
PATH=$(dirname ${BASH_SOURCE})/../bin:${PATH}
OCI=$(${BUILDAH_BINARY} info --format '{{.host.OCIRuntime}}' || command -v runc || command -v crun)
# Default timeout for a buildah command.
BUILDAH_TIMEOUT=${BUILDAH_TIMEOUT:-300}

# We don't invoke gnupg directly in many places, but this avoids ENOTTY errors
# when we invoke it directly in batch mode, and CI runs us without a terminal
# attached.
export GPG_TTY=/dev/null

function setup(){
    setup_tests
}

function setup_tests() {
    pushd "$(dirname "$(readlink -f "$BASH_SOURCE")")"

    # buildah/podman: "repository name must be lowercase".
    # me: "but it's a local file path, not a repository name!"
    # buildah/podman: "i dont care. no caps anywhere!"
    TESTDIR=$(mktemp -d --dry-run --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} buildah_tests.XXXXXX | tr A-Z a-z)
    mkdir --mode=0700 $TESTDIR

    mkdir -p ${TESTDIR}/{root,runroot,sigstore,registries.d}
    cat >${TESTDIR}/registries.d/default.yaml <<EOF
default-docker:
  sigstore-staging: file://${TESTDIR}/sigstore
docker:
  registry.access.redhat.com:
    sigstore: https://access.redhat.com/webassets/docker/content/sigstore
  registry.redhat.io:
    sigstore: https://registry.redhat.io/containers/sigstore
EOF

    # Common options for all buildah and podman invocations
    ROOTDIR_OPTS="--root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER}"
    BUILDAH_REGISTRY_OPTS="--registries-conf ${TESTSDIR}/registries.conf --registries-conf-dir ${TESTDIR}/registries.d --short-name-alias-conf ${TESTDIR}/cache/shortnames.conf"
    PODMAN_REGISTRY_OPTS="--registries-conf ${TESTSDIR}/registries.conf"
}

function starthttpd() {
    pushd ${2:-${TESTDIR}} > /dev/null
    go build -o serve ${TESTSDIR}/serve/serve.go
    portfile=$(mktemp)
    if test -z "${portfile}"; then
        echo error creating temporaty file
        exit 1
    fi
    ./serve ${1:-${BATS_TMPDIR}} 0 ${portfile} &
    HTTP_SERVER_PID=$!
    waited=0
    while ! test -s ${portfile} ; do
        sleep 0.1
        if test $((++waited)) -ge 300 ; then
            echo test http server did not start within timeout
            exit 1
        fi
    done
    HTTP_SERVER_PORT=$(cat ${portfile})
    rm -f ${portfile}
    popd > /dev/null
}

function stophttpd() {
    if test -n "$HTTP_SERVER_PID" ; then
        kill -HUP ${HTTP_SERVER_PID}
        unset HTTP_SERVER_PID
        unset HTTP_SERVER_PORT
    fi
    true
}

function teardown(){
    teardown_tests
}

function teardown_tests() {
    stophttpd
    stop_git_daemon

    # Workaround for #1991 - buildah + overlayfs leaks mount points.
    # Many tests leave behind /var/tmp/.../root/overlay and sub-mounts;
    # let's find those and clean them up, otherwise 'rm -rf' fails.
    # 'sort -r' guarantees that we umount deepest subpaths first.
    mount |\
        awk '$3 ~ testdir { print $3 }' testdir="^${TESTDIR}/" |\
        sort -r |\
        xargs --no-run-if-empty --max-lines=1 umount

    rm -fr ${TESTDIR}

    popd
}

function normalize_image_name() {
    for img in "$@"; do
        if [[ "${img##*/}" == "$img" ]] ; then
            echo -n docker.io/library/"$img"
        elif [[ docker.io/"${img##*/}" == "$img" ]] ; then
            echo -n docker.io/library/"${img##*/}"
        else
            echo -n "$img"
        fi
    done
}

function _prefetch() {
    if [ -z "${_BUILDAH_IMAGE_CACHEDIR}" ]; then
        _pgid=$(sed -ne 's/^NSpgid:\s*//p' /proc/$$/status)
        export _BUILDAH_IMAGE_CACHEDIR=${BATS_TMPDIR}/buildah-image-cache.$_pgid
        mkdir -p ${_BUILDAH_IMAGE_CACHEDIR}
    fi

    for img in "$@"; do
        img=$(normalize_image_name "$img")
        echo "# [checking for: $img]" >&2
        fname=$(tr -c a-zA-Z0-9.- - <<< "$img")
        if [ -d $_BUILDAH_IMAGE_CACHEDIR/$fname ]; then
            echo "# [restoring from cache: $_BUILDAH_IMAGE_CACHEDIR / $img]" >&2
            copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$img"
        else
            rm -fr $_BUILDAH_IMAGE_CACHEDIR/$fname
            echo "# [copy docker://$img dir:$_BUILDAH_IMAGE_CACHEDIR/$fname]" >&2
            for attempt in $(seq 3) ; do
                if copy docker://"$img" dir:$_BUILDAH_IMAGE_CACHEDIR/$fname ; then
                    break
                fi
                sleep 5
            done
            echo "# [copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:$img]" >&2
            copy dir:$_BUILDAH_IMAGE_CACHEDIR/$fname containers-storage:"$img"
        fi
    done
}

function createrandom() {
    dd if=/dev/urandom bs=1 count=${2:-256} of=${1:-${BATS_TMPDIR}/randomfile} status=none
}

function buildah() {
    ${BUILDAH_BINARY} ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
}

function imgtype() {
    ${IMGTYPE_BINARY} ${ROOTDIR_OPTS} "$@"
}

function copy() {
    ${COPY_BINARY} --max-parallel-downloads=1 ${ROOTDIR_OPTS} ${BUILDAH_REGISTRY_OPTS} "$@"
}

function podman() {
    command podman ${PODMAN_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
}

# There are various scenarios where we would like to execute `tests` as rootless user, however certain commands like `buildah mount`
# do not work in rootless session since a normal user cannot mount a filesystem unless they're in a user namespace along with its
# own mount namespace. In order to run such specific commands from a rootless session we must perform `buildah unshare`.
# Following function makes sure that invoked command is triggered inside a `buildah unshare` session if env is rootless.
function run_unshared() {
    if is_rootless; then
        $BUILDAH_BINARY unshare "$@"
    else
        command "$@"
    fi
}

function mkdir() {
    run_unshared mkdir "$@"
}

function touch() {
    run_unshared touch "$@"
}

function cp() {
    run_unshared cp "$@"
}

function rm() {
    run_unshared rm "$@"
}


#################
#  run_buildah  #  Invoke buildah, with timeout, using BATS 'run'
#################
#
# This is the preferred mechanism for invoking buildah:
#
#  * we use 'timeout' to abort (with a diagnostic) if something
#    takes too long; this is preferable to a CI hang.
#  * we log the command run and its output. This doesn't normally
#    appear in BATS output, but it will if there's an error.
#  * we check exit status. Since the normal desired code is 0,
#    that's the default; but the first argument can override:
#
#     run_buildah 125  nonexistent-subcommand
#     run_buildah '?'  some-other-command       # let our caller check status
#
# Since we use the BATS 'run' mechanism, $output and $status will be
# defined for our caller.
#
function run_buildah() {
    # Number as first argument = expected exit code; default 0
    # --retry as first argument = retry 3 times on error (eg registry flakes)
    local expected_rc=0
    local retry=1
    case "$1" in
        [0-9])           expected_rc=$1; shift;;
        [1-9][0-9])      expected_rc=$1; shift;;
        [12][0-9][0-9])  expected_rc=$1; shift;;
        '?')             expected_rc=  ; shift;;  # ignore exit code
        --retry)         retry=3;        shift;;  # retry network flakes
    esac

    # Remember command args, for possible use in later diagnostic messages
    MOST_RECENT_BUILDAH_COMMAND="buildah $*"

    # If session is rootless and `buildah mount` is invoked, perform unshare,
    # since normal user cannot mount a filesystem unless they're in a user namespace along with its own mount namespace.
    if is_rootless; then
        if [[ "$1" =~ mount ]]; then
            set "unshare" "$BUILDAH_BINARY" ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
        fi
    fi

    while [ $retry -gt 0 ]; do
        retry=$(( retry - 1 ))

        # stdout is only emitted upon error; this echo is to help a debugger
        echo "\$ $BUILDAH_BINARY $*"
        run env CONTAINERS_CONF=${CONTAINERS_CONF:-$(dirname ${BASH_SOURCE})/containers.conf} timeout --foreground --kill=10 $BUILDAH_TIMEOUT ${BUILDAH_BINARY} ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@"
        # without "quotes", multiple lines are glommed together into one
        if [ -n "$output" ]; then
            echo "$output"
{'fraction_non_alphanumeric': 0.14037424674912782, 'fraction_numerical': 0.008816999682841739, 'mean_word_length': 2.8870808678500985, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4468514', 'n_tokens_mistral': 5471, 'n_tokens_neox': 5290, 'n_words': 1422}