<commit_before>#!/bin/sh
# Copyright (C) 2014 <NAME> <<EMAIL>>

set -e

EMERGE_ROOT="/emerge-root"
EMERGE_BIN="${EMERGE_BIN:-emerge}"
CONFIG="/config"
CONFIG_TMP="${CONFIG}/tmp"
ROOTFS_BACKUP="/backup-rootfs"
PACKAGE_INSTALLED="${ROOTFS_BACKUP}/package.installed"
DOC_PACKAGE_INSTALLED="${ROOTFS_BACKUP}/doc.package.installed"
DOC_PACKAGE_PROVIDED="${ROOTFS_BACKUP}/doc.package.provided"
DOC_FOOTER_PURGED="${ROOTFS_BACKUP}/doc.footer.purged"
DOC_FOOTER_INCLUDES="${ROOTFS_BACKUP}/doc.footer.includes"

[ "$1" ] || {
    echo "--> Error: Empty repo id"
    exit 1
}

REPO="${1}"
IMAGE_NS="${REPO%%/*}"
IMAGE_ID="${REPO##*/}"

copy_gcc_libs() {
    LIBGCC="$(find /usr/lib/ -name libgcc_s.so.1)"
    LIBSTDC="$(find /usr/lib/ -name libstdc++.so.6)"

    for lib in $LIBGCC $LIBSTDC; do
        cp $lib $EMERGE_ROOT/lib64/
    done
}

extract_build_dependencies() {
    RESOURCE_SUFFIX="${1}"
    RESOURCE_VAR="${RESOURCE_SUFFIX}_FROM"
    if [ -n "${!RESOURCE_VAR}" ]; then
        for PARENT_REPO in ${!RESOURCE_VAR}; do
            if [ -f "${ROOTFS_BACKUP}/${PARENT_REPO/\//_}-${RESOURCE_SUFFIX}.tar" ]; then
                tar xpf "${ROOTFS_BACKUP}/${PARENT_REPO/\//_}-${RESOURCE_SUFFIX}.tar"
            fi
        done
    fi
}

generate_documentation_footer() {
    echo "#### Purged" > "${DOC_FOOTER_PURGED}"
    write_checkbox_line "Headers" "${KEEP_HEADERS}" "${DOC_FOOTER_PURGED}" "negate"
    write_checkbox_line "Static Libs" "${KEEP_STATIC_LIBS}" "${DOC_FOOTER_PURGED}" "negate"
    if [[ -n $HEADERS_FROM ]] || [[ -n $STATIC_LIBS_FROM ]] || [[ -n $ICONV_FROM ]]; then
        echo -e "\n#### Included" > "${DOC_FOOTER_INCLUDES}"
        if [[ -n $HEADERS_FROM ]]; then
            write_checkbox_line "Headers from ${HEADERS_FROM}" "checked" "${DOC_FOOTER_INCLUDES}"
        fi
        if [[ -n $STATIC_LIBS_FROM ]]; then
            write_checkbox_line "Static Libs from ${STATIC_LIBS_FROM}" "checked" "${DOC_FOOTER_INCLUDES}"
        fi
        if [[ -n $ICONV_FROM ]]; then
            write_checkbox_line "Glibc Iconv Encodings" "checked" "${DOC_FOOTER_INCLUDES}"
        fi
    fi
}

generate_documentation() {
    DOC_FILE="${CONFIG}/PACKAGES.md"
    TABLE_HEADER="Package | USE Flags\n--------|----------"
    echo "#### Installed" > $DOC_FILE
    if [[ -f ${DOC_PACKAGE_INSTALLED} ]]; then
        echo -e "$TABLE_HEADER" >> $DOC_FILE
        cat ${DOC_PACKAGE_INSTALLED} | sed -e "1d" >> $DOC_FILE
    else
        echo "None." >> $DOC_FILE
    fi
    echo "#### Inherited" >> $DOC_FILE
    echo -e "$TABLE_HEADER" >> $DOC_FILE
    if [[ -f ${DOC_PACKAGE_PROVIDED} ]]; then
        cat ${DOC_PACKAGE_PROVIDED} >> $DOC_FILE
    else
        echo "**FROM scratch** |" >> $DOC_FILE
    fi
    if [[ -f ${DOC_FOOTER_PURGED} ]]; then
        cat ${DOC_FOOTER_PURGED} >> $DOC_FILE
    fi
    if [[ -f ${DOC_FOOTER_INCLUDES} ]]; then
        cat ${DOC_FOOTER_INCLUDES} >> $DOC_FILE
    fi
}

# Appends a github markdown line with a checkbox and label to given file.
#
# Arguments:
# 1: checkbox label
# 2: is checked
# 3: file
# 4: negate checked state, when set the true/false eval of $2 is negated, optional
write_checkbox_line() {
    LABEL="${1}"
    CHECKED="${2}"
    NEGATE_CHECK_STATE="${4}"
    if [[ -z "$CHECKED" ]] || [[ "$CHECKED" = "false" ]]; then
        STATE=0
    else 
        STATE=1
    fi
    if [[ -n $NEGATE_CHECK_STATE ]]; then
        if [[ "$STATE" == 1 ]]; then
            STATE=0
        else 
            STATE=1
        fi
    fi
    if [[ "$STATE" == 1 ]]; then
        CHECKBOX="- [x]"
    else 
        CHECKBOX="- [ ]"
    fi
    echo "${CHECKBOX} ${LABEL}" >> "${3}"
}

# Adds a package entry in $DOC_PACKAGE_INSTALLED to document non-Portage package installs.
# You should only use this function from the finish_rootfs_build() hook.
#
# Arguments:
# 1: package group (for example "gem" if you installed ruby gems)
# 2: package-version
# 3: optional string that appears in the use flags column
log_as_installed() {
    echo "*${1}*: ${2} | ${3}" >> "${DOC_PACKAGE_INSTALLED}"
}

install_docker_gen() {
    wget http://github.com/jwilder/docker-gen/releases/download/0.3.2/docker-gen-linux-amd64-0.3.2.tar.gz
    mkdir -p $EMERGE_ROOT/bin
    tar -C $EMERGE_ROOT/bin -xvzf docker-gen-linux-amd64-0.3.2.tar.gz
    mkdir -p $EMERGE_ROOT/config/template
    log_as_installed "manual install" "docker-gen-0.3.2" "http://github.com/jwilder/docker-gen/"
}

mkdir -p $EMERGE_ROOT

# read config, mounted via build.sh
[[ -f ${CONFIG}/Buildconfig.sh ]] && source ${CONFIG}/Buildconfig.sh || :

# call configure bob hook if declared in Buildconfig.sh
declare -F configure_bob &>/dev/null && configure_bob

if [ -n "$PACKAGES" ]; then
    mkdir -p ${ROOTFS_BACKUP} 

    # set ROOT env for emerge calls
    export ROOT="${EMERGE_ROOT}"

    # call pre install hook if declared in Buildconfig.sh
    declare -F configure_rootfs_build &>/dev/null && configure_rootfs_build

    # generate installed package list
    "${EMERGE_BIN}" -p $PACKAGES | grep -Eow "\[.*\] (.*) to" | awk '{print $(NF-1)}' > ${PACKAGE_INSTALLED}

    [[ -f ${DOC_PACKAGE_INSTALLED} ]] && \
        echo -e "$(cat ${DOC_PACKAGE_INSTALLED})\n$(cat ${DOC_PACKAGE_PROVIDED})" > ${DOC_PACKAGE_PROVIDED}

    echo "**FROM ${REPO}** |" > ${DOC_PACKAGE_INSTALLED}
    # generate installed package list with use flags for auto docs
    "${EMERGE_BIN}" -p $PACKAGES | perl -nle 'print "$1 | `$3`" if /\[.*\] (.*) to \/.*\/( USE=")?([a-z0-9\- (){}]*)?/' | \
        sed /^virtual/d | sort -u >> "${DOC_PACKAGE_INSTALLED}"

    # install packages (defined via Buildconfig.sh)
    "${EMERGE_BIN}" -v baselayout $PACKAGES

    cat ${PACKAGE_INSTALLED} | sed -e /^virtual/d >> /etc/portage/profile/package.provided

    # backup headers and static files, depending images can pull them in again
    if [ -d $EMERGE_ROOT/usr/include ]; then 
        find $EMERGE_ROOT/usr/include -type f -name '*.h' | tar -cpf ${ROOTFS_BACKUP}/${IMAGE_NS}_${IMAGE_ID}-HEADERS.tar --files-from -
    fi
    if [ -d $EMERGE_ROOT/usr/lib64 ]; then
        find $EMERGE_ROOT/usr/lib64 -type f -name '*.a' | tar -cpf ${ROOTFS_BACKUP}/${IMAGE_NS}_${IMAGE_ID}-STATIC_LIBS.tar --files-from -
    fi

    # extract any possible required headers and static libs from previous builds
    for resource in "HEADERS" "STATIC_LIBS" "ICONV"; do
        extract_build_dependencies ${resource}
    done

    # handle bug in portage when using custom root, user/groups created during install are not created at the custom root but on the host<commit_msg>Fix package.installed header. Add missing file check for package.installed.<commit_after>#!/bin/sh
# Copyright (C) 2014 <NAME> <<EMAIL>>

set -e

EMERGE_ROOT="/emerge-root"
EMERGE_BIN="${EMERGE_BIN:-emerge}"
CONFIG="/config"
CONFIG_TMP="${CONFIG}/tmp"
ROOTFS_BACKUP="/backup-rootfs"
PACKAGE_INSTALLED="${ROOTFS_BACKUP}/package.installed"
DOC_PACKAGE_INSTALLED="${ROOTFS_BACKUP}/doc.package.installed"
DOC_PACKAGE_PROVIDED="${ROOTFS_BACKUP}/doc.package.provided"
DOC_FOOTER_PURGED="${ROOTFS_BACKUP}/doc.footer.purged"
DOC_FOOTER_INCLUDES="${ROOTFS_BACKUP}/doc.footer.includes"

[ "$1" ] || {
    echo "--> Error: Empty repo id"
    exit 1
}

REPO="${1}"
IMAGE_NS="${REPO%%/*}"
IMAGE_ID="${REPO##*/}"

copy_gcc_libs() {
    LIBGCC="$(find /usr/lib/ -name libgcc_s.so.1)"
    LIBSTDC="$(find /usr/lib/ -name libstdc++.so.6)"

    for lib in $LIBGCC $LIBSTDC; do
        cp $lib $EMERGE_ROOT/lib64/
    done
}

extract_build_dependencies() {
    RESOURCE_SUFFIX="${1}"
    RESOURCE_VAR="${RESOURCE_SUFFIX}_FROM"
    if [ -n "${!RESOURCE_VAR}" ]; then
        for PARENT_REPO in ${!RESOURCE_VAR}; do
            if [ -f "${ROOTFS_BACKUP}/${PARENT_REPO/\//_}-${RESOURCE_SUFFIX}.tar" ]; then
                tar xpf "${ROOTFS_BACKUP}/${PARENT_REPO/\//_}-${RESOURCE_SUFFIX}.tar"
            fi
        done
    fi
}

generate_documentation_footer() {
    echo "#### Purged" > "${DOC_FOOTER_PURGED}"
    write_checkbox_line "Headers" "${KEEP_HEADERS}" "${DOC_FOOTER_PURGED}" "negate"
    write_checkbox_line "Static Libs" "${KEEP_STATIC_LIBS}" "${DOC_FOOTER_PURGED}" "negate"
    if [[ -n $HEADERS_FROM ]] || [[ -n $STATIC_LIBS_FROM ]] || [[ -n $ICONV_FROM ]]; then
        echo -e "\n#### Included" > "${DOC_FOOTER_INCLUDES}"
        if [[ -n $HEADERS_FROM ]]; then
            write_checkbox_line "Headers from ${HEADERS_FROM}" "checked" "${DOC_FOOTER_INCLUDES}"
        fi
        if [[ -n $STATIC_LIBS_FROM ]]; then
            write_checkbox_line "Static Libs from ${STATIC_LIBS_FROM}" "checked" "${DOC_FOOTER_INCLUDES}"
        fi
        if [[ -n $ICONV_FROM ]]; then
            write_checkbox_line "Glibc Iconv Encodings" "checked" "${DOC_FOOTER_INCLUDES}"
        fi
    fi
}

generate_documentation() {
    DOC_FILE="${CONFIG}/PACKAGES.md"
    TABLE_HEADER="Package | USE Flags\n--------|----------"
    echo "#### Installed" > $DOC_FILE
    if [[ -f ${DOC_PACKAGE_INSTALLED} ]]; then
        echo -e "$TABLE_HEADER" >> $DOC_FILE
        cat ${DOC_PACKAGE_INSTALLED} | sed -e "1d" >> $DOC_FILE
    else
        echo "None." >> $DOC_FILE
    fi
    echo "#### Inherited" >> $DOC_FILE
    echo -e "$TABLE_HEADER" >> $DOC_FILE
    if [[ -f ${DOC_PACKAGE_PROVIDED} ]]; then
        cat ${DOC_PACKAGE_PROVIDED} >> $DOC_FILE
    else
        echo "**FROM scratch** |" >> $DOC_FILE
    fi
    if [[ -f ${DOC_FOOTER_PURGED} ]]; then
        cat ${DOC_FOOTER_PURGED} >> $DOC_FILE
    fi
    if [[ -f ${DOC_FOOTER_INCLUDES} ]]; then
        cat ${DOC_FOOTER_INCLUDES} >> $DOC_FILE
    fi
}

# Appends a github markdown line with a checkbox and label to given file.
#
# Arguments:
# 1: checkbox label
# 2: is checked
# 3: file
# 4: negate checked state, when set the true/false eval of $2 is negated, optional
write_checkbox_line() {
    LABEL="${1}"
    CHECKED="${2}"
    NEGATE_CHECK_STATE="${4}"
    if [[ -z "$CHECKED" ]] || [[ "$CHECKED" = "false" ]]; then
        STATE=0
    else 
        STATE=1
    fi
    if [[ -n $NEGATE_CHECK_STATE ]]; then
        if [[ "$STATE" == 1 ]]; then
            STATE=0
        else 
            STATE=1
        fi
    fi
    if [[ "$STATE" == 1 ]]; then
        CHECKBOX="- [x]"
    else 
        CHECKBOX="- [ ]"
    fi
    echo "${CHECKBOX} ${LABEL}" >> "${3}"
}

# Adds a package entry in $DOC_PACKAGE_INSTALLED to document non-Portage package installs.
# You should only use this function from the finish_rootfs_build() hook.
#
# Arguments:
# 1: package group (for example "gem" if you installed ruby gems)
# 2: package-version
# 3: optional string that appears in the use flags column
log_as_installed() {
    echo "*${1}*: ${2} | ${3}" >> "${DOC_PACKAGE_INSTALLED}"
}

install_docker_gen() {
    wget http://github.com/jwilder/docker-gen/releases/download/0.3.2/docker-gen-linux-amd64-0.3.2.tar.gz
    mkdir -p $EMERGE_ROOT/bin
    tar -C $EMERGE_ROOT/bin -xvzf docker-gen-linux-amd64-0.3.2.tar.gz
    mkdir -p $EMERGE_ROOT/config/template
    log_as_installed "manual install" "docker-gen-0.3.2" "http://github.com/jwilder/docker-gen/"
}

mkdir -p $EMERGE_ROOT

# read config, mounted via build.sh
[[ -f ${CONFIG}/Buildconfig.sh ]] && source ${CONFIG}/Buildconfig.sh || :

# call configure bob hook if declared in Buildconfig.sh
declare -F configure_bob &>/dev/null && configure_bob

if [ -n "$PACKAGES" ]; then
    mkdir -p ${ROOTFS_BACKUP} 

    # set ROOT env for emerge calls
    export ROOT="${EMERGE_ROOT}"

    # call pre install hook if declared in Buildconfig.sh
    declare -F configure_rootfs_build &>/dev/null && configure_rootfs_build

    # generate installed package list
    "${EMERGE_BIN}" -p $PACKAGES | grep -Eow "\[.*\] (.*) to" | awk '{print $(NF-1)}' > ${PACKAGE_INSTALLED}

    [[ -f ${DOC_PACKAGE_INSTALLED} ]] && \
        echo -e "$(cat ${DOC_PACKAGE_INSTALLED})\n$(cat ${DOC_PACKAGE_PROVIDED})" > ${DOC_PACKAGE_PROVIDED}

    echo "**FROM ${REPO/\images\//}** |" > ${DOC_PACKAGE_INSTALLED}
    # generate installed package list with use flags for auto docs
    "${EMERGE_BIN}" -p $PACKAGES | perl -nle 'print "$1 | `$3`" if /\[.*\] (.*) to \/.*\/( USE=")?([a-z0-9\- (){}]*)?/' | \
        sed /^virtual/d | sort -u >> "${DOC_PACKAGE_INSTALLED}"

    # install packages (defined via Buildconfig.sh)
    "${EMERGE_BIN}" -v baselayout $PACKAGES

    [[ -f ${PACKAGE_INSTALLED} ]] && cat ${PACKAGE_INSTALLED} | sed -e /^virtual/d >> /etc/portage/profile/package.provided

    # backup headers and static files, depending images can pull them in again
    if [ -d $EMERGE_ROOT/usr/include ]; then 
        find $EMERGE_ROOT/usr/include -type f -name '*.h' | tar -cpf ${ROOTFS_BACKUP}/${IMAGE_NS}_${IMAGE_ID}-HEADERS.tar --files-from -
    fi
    if [ -d $EMERGE_ROOT/usr/lib64 ]; then
        find $EMERGE_ROOT/usr/lib64 -type f -name '*.a' | tar -cpf ${ROOTFS_BACKUP}/${IMAGE_NS}_${IMAGE_ID}-STATIC_LIBS.tar --files-from -
    fi

    # extract any possible required headers and static libs from previous builds
    for resource in "HEADERS" "STATIC_LIBS" "ICONV"; do
        extract_build_dependencies ${resource}
    done

    # handle bug in portage when using custom root, user/groups created during install are not created at the custom root but on the host
{'fraction_non_alphanumeric': 0.16425745587340232, 'fraction_numerical': 0.008977480219111381, 'mean_word_length': 3.110381488430269, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5734001', 'n_tokens_mistral': 5006, 'n_tokens_neox': 4674, 'n_words': 1105}