<commit_before>#!/usr/bin/env bash
#
# Copyright (c) 2014-2017, <NAME> <<EMAIL>>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
#    disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
# following disclaimer in the documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

DOCKER_IO=$(command -v docker.io)
DOCKER="${DOCKER:-${DOCKER_IO:-docker}}"
DOCKER_BUILD_OPTS="${DOCKER_BUILD_OPTS:-}"

_container_mount_portage="false"

# Is this engine functional? Called once per engine in current image dependency graph.
function validate_engine() {
    local docker_version
    _required_binaries+=" docker"
    has_required_binaries
    docker_version=$(${DOCKER} "version") || die "Failed to query the docker daemon:\n${docker_version}"
}

# Has given image_id all requirements to start the build? Called once per image in current image dependency graph.
#
# Arguments:
# 1: image_id (i.e. kubler/busybox)
# 2: image_type ($_IMAGE_PATH or $_BUILDER_PATH)
function validate_image() {
    local image_id image_type
    image_id="$1"
    image_type="$2"
    expand_image_id "${image_id}" "${image_type}"
    # shellcheck disable=SC2154
    file_exists_or_die "${__expand_image_id}/Dockerfile.template"
}

# Generate Dockerfile from Dockerfile.template
#
# Arguments:
# 1: image_path (i.e. kubler/images/busybox)
function generate_dockerfile() {
    local image_path sed_param bob_var
    image_path="$1"
    sed_param=()
    # also make variables starting with BOB_ available in Dockerfile.template
    for bob_var in ${!BOB_*}; do
        sed_param+=(-e "s|\${${bob_var}}|${!bob_var}|")
    done
    # shellcheck disable=SC2016,SC2153,SC2154
    sed "${sed_param[@]}" \
        -e 's|${IMAGE_PARENT}|'"${IMAGE_PARENT}"'|g' \
        -e 's|${DEFAULT_BUILDER}|'"${DEFAULT_BUILDER}"'|g' \
        -e 's/${NAMESPACE}/'"${_current_namespace}"'/g' \
        -e 's/${TAG}/'"${IMAGE_TAG}"'/g' \
        -e 's/${MAINTAINER}/'"${AUTHOR}"'/g' \
        "${image_path}/Dockerfile.template" > "${image_path}/Dockerfile" \
            || die "error while generating ${image_path}/Dockerfile"
}

# Returns given tag value from dockerfile or exit signal 3 if tag was not found.
# Returns "true" if TAG was found but has no value.
#
# Arguments:
# 1: tag (i.e. FROM)
# 2: image_path (i.e. kubler/images/busybox)
function get_dockerfile_tag() {
    __get_dockerfile_tag=
    local tag image_path dockerfile grep_out regex
    tag="$1"
    image_path="$2"
    dockerfile="${image_path}/Dockerfile"
    file_exists_or_die "${dockerfile}"
    grep_out="$(grep ^"${tag}" "${dockerfile}")"
    regex="^${tag} ?(.*)?"
    if [[ "${grep_out}" =~ $regex ]]; then
        if [[ "${BASH_REMATCH[1]}" ]]; then
            # shellcheck disable=SC2034
            __get_dockerfile_tag="${BASH_REMATCH[1]}"
        else
            exit 3
        fi
    fi
}

# Remove image from local image store
#
# Arguments:
#
# 1: image_id
# 2: image_tag
function remove_image() {
    local image_id image_tag
    image_id="$1"
    image_tag="${2:-${IMAGE_TAG}}"
    "${DOCKER}" rmi -f "${image_id}:${image_tag}" || die "failed to remove image ${image_id}:${image_tag}"
}

# Build the image for given image_id
#
# Arguments:
# 1: image_id (i.e. kubler/busybox)
# 2: image_type - $_IMAGE_PATH or $_BUILDER_PATH, defaults to $_IMAGE_PATH
function build_image() {
    local image_id image_type image_expanded builder_id builder_commit_id current_image bob_var run_id
    image_id="${1}"
    image_type="${2:-${_IMAGE_PATH}}"
    expand_image_id "${image_id}" "${image_type}"
    image_expanded="${__expand_image_id}"

    msg "--> build image ${image_id}"
    image_exists_or_rm "${image_id}" "${image_type}"
    local exists_return=$?
    [[ ${exists_return} -eq 0 ]] && return 0
    # if the builder image does not exist we need to ensure there is no pre-existing rootfs.tar
    if [[ ${exists_return} -eq 3 && "${image_type}" == "${_BUILDER_PATH}" ]]; then
        [[ -f "${image_expanded}/rootfs.tar" ]] && rm "${image_expanded}/rootfs.tar"
    fi

    generate_dockerfile "${image_expanded}"

    # build rootfs?
    # shellcheck disable=SC2154
    if [[ ! -f "${image_expanded}/rootfs.tar" || "${_arg_force_full_image_build}" == 'on' ]] && \
       [[ "${image_type}" == "${_IMAGE_PATH}" || "${image_id}" != "${_current_namespace}"/*-core ]]; then

        msg "--> phase 1: building root fs"

        get_build_container "${image_id}" "${image_type}"
        builder_id="${__get_build_container}"

        # determine build container commit id
        builder_commit_id=""
        current_image=${image_id##*/}
        if [[ ! -z "${BUILDER}" ]]; then
            builder_commit_id="${BUILDER##*/}-${current_image}"
        elif [[ "${image_type}" == "${_IMAGE_PATH}" ]]; then
            builder_commit_id="${DEFAULT_BUILDER##*/}-${current_image}"
        fi

        if [[ "${image_type}" == "${_BUILDER_PATH}" ]]; then
            [[ "${builder_id}" == "${image_id}" && "${image_id}" != "${_current_namespace}"/*-core ]] && \
                builder_id="${image_id}-core"
            builder_commit_id="${image_id##*/}"
        fi

        local config_dir
        # mounts for build container
        get_absolute_path "${image_expanded}"
        # shellcheck disable=SC2154
        config_dir="${__get_absolute_path}"
        _container_mounts=("${config_dir}:/config"
                           "${_KUBLER_DIR}/tmp/distfiles:/distfiles"
                           "${_KUBLER_DIR}/tmp/packages:/packages"
                          )
        # shellcheck disable=SC2034
        BOB_CURRENT_TARGET="${image_id}"

        # pass variables starting with BOB_ to build container as ENV
        for bob_var in ${!BOB_*}; do
            _container_env+=("${bob_var}=${!bob_var}")
        done

        _container_cmd=("kubler-build-root")
        _container_mount_portage="true"

        msg "using ${builder_id}:${IMAGE_TAG}"<commit_msg>Fix wrong builder_commit_id when mixing build containers #87<commit_after>#!/usr/bin/env bash
#
# Copyright (c) 2014-2017, <NAME> <<EMAIL>>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
#    disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
# following disclaimer in the documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

DOCKER_IO=$(command -v docker.io)
DOCKER="${DOCKER:-${DOCKER_IO:-docker}}"
DOCKER_BUILD_OPTS="${DOCKER_BUILD_OPTS:-}"

_container_mount_portage="false"

# Is this engine functional? Called once per engine in current image dependency graph.
function validate_engine() {
    local docker_version
    _required_binaries+=" docker"
    has_required_binaries
    docker_version=$(${DOCKER} "version") || die "Failed to query the docker daemon:\n${docker_version}"
}

# Has given image_id all requirements to start the build? Called once per image in current image dependency graph.
#
# Arguments:
# 1: image_id (i.e. kubler/busybox)
# 2: image_type ($_IMAGE_PATH or $_BUILDER_PATH)
function validate_image() {
    local image_id image_type
    image_id="$1"
    image_type="$2"
    expand_image_id "${image_id}" "${image_type}"
    # shellcheck disable=SC2154
    file_exists_or_die "${__expand_image_id}/Dockerfile.template"
}

# Generate Dockerfile from Dockerfile.template
#
# Arguments:
# 1: image_path (i.e. kubler/images/busybox)
function generate_dockerfile() {
    local image_path sed_param bob_var
    image_path="$1"
    sed_param=()
    # also make variables starting with BOB_ available in Dockerfile.template
    for bob_var in ${!BOB_*}; do
        sed_param+=(-e "s|\${${bob_var}}|${!bob_var}|")
    done
    # shellcheck disable=SC2016,SC2153,SC2154
    sed "${sed_param[@]}" \
        -e 's|${IMAGE_PARENT}|'"${IMAGE_PARENT}"'|g' \
        -e 's|${DEFAULT_BUILDER}|'"${DEFAULT_BUILDER}"'|g' \
        -e 's/${NAMESPACE}/'"${_current_namespace}"'/g' \
        -e 's/${TAG}/'"${IMAGE_TAG}"'/g' \
        -e 's/${MAINTAINER}/'"${AUTHOR}"'/g' \
        "${image_path}/Dockerfile.template" > "${image_path}/Dockerfile" \
            || die "error while generating ${image_path}/Dockerfile"
}

# Returns given tag value from dockerfile or exit signal 3 if tag was not found.
# Returns "true" if TAG was found but has no value.
#
# Arguments:
# 1: tag (i.e. FROM)
# 2: image_path (i.e. kubler/images/busybox)
function get_dockerfile_tag() {
    __get_dockerfile_tag=
    local tag image_path dockerfile grep_out regex
    tag="$1"
    image_path="$2"
    dockerfile="${image_path}/Dockerfile"
    file_exists_or_die "${dockerfile}"
    grep_out="$(grep ^"${tag}" "${dockerfile}")"
    regex="^${tag} ?(.*)?"
    if [[ "${grep_out}" =~ $regex ]]; then
        if [[ "${BASH_REMATCH[1]}" ]]; then
            # shellcheck disable=SC2034
            __get_dockerfile_tag="${BASH_REMATCH[1]}"
        else
            exit 3
        fi
    fi
}

# Remove image from local image store
#
# Arguments:
#
# 1: image_id
# 2: image_tag
function remove_image() {
    local image_id image_tag
    image_id="$1"
    image_tag="${2:-${IMAGE_TAG}}"
    "${DOCKER}" rmi -f "${image_id}:${image_tag}" || die "failed to remove image ${image_id}:${image_tag}"
}

# Build the image for given image_id
#
# Arguments:
# 1: image_id (i.e. kubler/busybox)
# 2: image_type - $_IMAGE_PATH or $_BUILDER_PATH, defaults to $_IMAGE_PATH
function build_image() {
    local image_id image_type image_expanded builder_id builder_commit_id current_image bob_var run_id
    image_id="${1}"
    image_type="${2:-${_IMAGE_PATH}}"
    expand_image_id "${image_id}" "${image_type}"
    image_expanded="${__expand_image_id}"

    msg "--> build image ${image_id}"
    image_exists_or_rm "${image_id}" "${image_type}"
    local exists_return=$?
    [[ ${exists_return} -eq 0 ]] && return 0
    # if the builder image does not exist we need to ensure there is no pre-existing rootfs.tar
    if [[ ${exists_return} -eq 3 && "${image_type}" == "${_BUILDER_PATH}" ]]; then
        [[ -f "${image_expanded}/rootfs.tar" ]] && rm "${image_expanded}/rootfs.tar"
    fi

    generate_dockerfile "${image_expanded}"

    # build rootfs?
    # shellcheck disable=SC2154
    if [[ ! -f "${image_expanded}/rootfs.tar" || "${_arg_force_full_image_build}" == 'on' ]] && \
       [[ "${image_type}" == "${_IMAGE_PATH}" || "${image_id}" != "${_current_namespace}"/*-core ]]; then

        msg "--> phase 1: building root fs"

        get_build_container "${image_id}" "${image_type}"
        builder_id="${__get_build_container}"

        # determine build container commit id
        builder_commit_id=""
        current_image=${image_id##*/}
        if [[ ! -z "${BUILDER}" ]]; then
            builder_commit_id="${BUILDER##*/}-${current_image}"
        elif [[ "${image_type}" == "${_IMAGE_PATH}" ]]; then
            builder_commit_id="${builder_id##*/}-${current_image}"
        fi

        if [[ "${image_type}" == "${_BUILDER_PATH}" ]]; then
            [[ "${builder_id}" == "${image_id}" && "${image_id}" != "${_current_namespace}"/*-core ]] && \
                builder_id="${image_id}-core"
            builder_commit_id="${image_id##*/}"
        fi

        local config_dir
        # mounts for build container
        get_absolute_path "${image_expanded}"
        # shellcheck disable=SC2154
        config_dir="${__get_absolute_path}"
        _container_mounts=("${config_dir}:/config"
                           "${_KUBLER_DIR}/tmp/distfiles:/distfiles"
                           "${_KUBLER_DIR}/tmp/packages:/packages"
                          )
        # shellcheck disable=SC2034
        BOB_CURRENT_TARGET="${image_id}"

        # pass variables starting with BOB_ to build container as ENV
        for bob_var in ${!BOB_*}; do
            _container_env+=("${bob_var}=${!bob_var}")
        done

        _container_cmd=("kubler-build-root")
        _container_mount_portage="true"

        msg "using ${builder_id}:${IMAGE_TAG}"
{'fraction_non_alphanumeric': 0.13962936564504633, 'fraction_numerical': 0.009836065573770493, 'mean_word_length': 3.5144787644787643, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2840696', 'n_tokens_mistral': 4609, 'n_tokens_neox': 4356, 'n_words': 1194}