<commit_before>#!/bin/sh

SCRIPTSDIR=$(dirname $0)
TOPDIR=$(realpath ${SCRIPTSDIR}/..)
. ${TOPDIR}/scripts/common_settings.sh
. ${TOPDIR}/scripts/helpers.sh

UPDATE_TAR="${BUILD_DIR}/update/rootfs.tar"
UPDATE_TAR_XZ="${UPDATE_TAR}.xz"

# init possible entries of a MANIFEST
FILENAME="container"
FILESIZE=""
FILETYPE=""
DESCRIPTION=""
VERSION=""
MD5SUM=""
KEY=""
FILESYSTEM_LIST="default.txt"

# print parameter list of this script
usage()
{
    echo "Usage: mk_container.sh <-n FILENAME> [-l ROOTFS-LIST] [-k KEY] [-d DESCRIPTION] [-v VERSION]"
    echo "    -n  name of the update packet with container to be packed"
    echo "    -l  use this file in the directory \"scripts/rootfs_lists/\" to fill the root file system of container"
    echo "    -k  use this RSA key in the directory \"scripts/keys/\" to encrypt the container"
    echo "    -d  use that description within MANIFEST"
    echo "    -v  use that version string within MANIFEST"
    echo ""
}

# create the MANIFEST file
print_manifest_entry()
{
    echo "FILENAME=${FILENAME}.tar.xz"
    echo "FILESIZE=${FILESIZE}"
    echo "FILETYPE=Container"
    echo "MD5SUM=${MD5SUM}"
    test -n "${KEY}" && echo "KEY=${KEY}.router"
    test -n "${VERSION}" && echo "VERSION=${VERSION}"
    test -n "${DESCRIPTION}" && echo "DESCRIPTION=${DESCRIPTION}"
    echo ""
}

# read all options from command line
get_options()
{
    while [ -n "${1}" ] ; do
        case "${1}" in
            "-n")
                shift
                FILENAME="${1}"
            ;;
            "-l")
                shift
                FILESYSTEM_LIST="${1}"
            ;;
            "-d")
                shift
                DESCRIPTION="${1}"
            ;;
            "-k")
                shift
                KEY="${1}"
            ;;
            "-v")
                shift
                VERSION="${1}"
            ;;
            "-h")
                shift
                usage
                exit 0
            ;;
            *)
                usage
            ;;
        esac
        shift
    done
}

# strips the file $1 if possible
do_strip()
{
    FILE_TYPE=$(file ${1})
    if echo "${FILE_TYPE}" | grep -q "${ELF_BINARY}" ; then
        if echo "${FILE_TYPE}" | grep -q "executable" ; then
            ${M3_CROSS_COMPILE}strip "${1}" > /dev/null 2>&1
        else
            if echo "${FILE_TYPE}" | grep -q "shared object" ; then
                ${M3_CROSS_COMPILE}strip "${1}" > /dev/null 2>&1
            else
                if echo "${FILE_TYPE}" | grep -q "BuildID" ; then
                    # kernel modules my not be fully stripped
                    ${M3_CROSS_COMPILE}strip --strip-debug "${1}"
                fi
            fi
        fi
    fi
}

# process the list of files and permissions for the filesystem
# $1 = action: create_tar, create_list
# $2 = fs list
# $3 = FS_OUTFILE
process_filesystem_list()
{
    ACTION="${1}"
    FILESYSTEM_LIST="${SCRIPTSDIR}/rootfs_lists/${2}"
    FS_OUTFILE="${3}"

    echo "-> Creating archive $FS_OUTFILE containing the root file system"
    if ! [ -e "${FILESYSTEM_LIST}" ] ; then
        echo "   ERROR: the given list file \"${FILESYSTEM_LIST}\" does not exist"
        exit 1
    fi

    rm -f "${FS_OUTFILE}"
    rm -rf "${FS_TARGET_DIR}"
    mkdir "${FS_TARGET_DIR}"

    test -d $(dirname ${FS_OUTFILE}) || mkdir -p $(dirname ${FS_OUTFILE})
    OLDIFS=$IFS
    IFS=$'\n'
    for LINE in $(cat ${FILESYSTEM_LIST} | grep -F -v '#' | grep -Ev '^$' ) ; do
        TYPE=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f1 | sed 's/^ *//' | sed 's/ *$//' )
        DESTINATION_FILE=$(eval echo ${LINE} | tr -s ' ' | cut -s -d' ' -f2 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_1=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f3 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_2=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f4 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_3=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f5 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_4=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f6 | sed 's/^ *//' | sed 's/ *$//' )
<commit_msg>Fixed FILESYSTEM_LIST<commit_after>#!/bin/sh

SCRIPTSDIR=$(dirname $0)
TOPDIR=$(realpath ${SCRIPTSDIR}/..)
. ${TOPDIR}/scripts/common_settings.sh
. ${TOPDIR}/scripts/helpers.sh

UPDATE_TAR="${BUILD_DIR}/update/rootfs.tar"
UPDATE_TAR_XZ="${UPDATE_TAR}.xz"

# init possible entries of a MANIFEST
FILENAME="container"
FILESIZE=""
FILETYPE=""
DESCRIPTION=""
VERSION=""
MD5SUM=""
KEY=""
FILESYSTEM_LIST="default.txt"

# print parameter list of this script
usage()
{
    echo "Usage: mk_container.sh <-n FILENAME> [-l ROOTFS-LIST] [-k KEY] [-d DESCRIPTION] [-v VERSION]"
    echo "    -n  name of the update packet with container to be packed"
    echo "    -l  use this file in the directory \"scripts/rootfs_lists/\" to fill the root file system of container"
    echo "    -k  use this RSA key in the directory \"scripts/keys/\" to encrypt the container"
    echo "    -d  use that description within MANIFEST"
    echo "    -v  use that version string within MANIFEST"
    echo ""
}

# create the MANIFEST file
print_manifest_entry()
{
    echo "FILENAME=${FILENAME}.tar.xz"
    echo "FILESIZE=${FILESIZE}"
    echo "FILETYPE=Container"
    echo "MD5SUM=${MD5SUM}"
    test -n "${KEY}" && echo "KEY=${KEY}.router"
    test -n "${VERSION}" && echo "VERSION=${VERSION}"
    test -n "${DESCRIPTION}" && echo "DESCRIPTION=${DESCRIPTION}"
    echo ""
}

# read all options from command line
get_options()
{
    while [ -n "${1}" ] ; do
        case "${1}" in
            "-n")
                shift
                FILENAME="${1}"
            ;;
            "-l")
                shift
                FILESYSTEM_LIST="${1}"
            ;;
            "-d")
                shift
                DESCRIPTION="${1}"
            ;;
            "-k")
                shift
                KEY="${1}"
            ;;
            "-v")
                shift
                VERSION="${1}"
            ;;
            "-h")
                shift
                usage
                exit 0
            ;;
            *)
                usage
            ;;
        esac
        shift
    done
}

# strips the file $1 if possible
do_strip()
{
    FILE_TYPE=$(file ${1})
    if echo "${FILE_TYPE}" | grep -q "${ELF_BINARY}" ; then
        if echo "${FILE_TYPE}" | grep -q "executable" ; then
            ${M3_CROSS_COMPILE}strip "${1}" > /dev/null 2>&1
        else
            if echo "${FILE_TYPE}" | grep -q "shared object" ; then
                ${M3_CROSS_COMPILE}strip "${1}" > /dev/null 2>&1
            else
                if echo "${FILE_TYPE}" | grep -q "BuildID" ; then
                    # kernel modules my not be fully stripped
                    ${M3_CROSS_COMPILE}strip --strip-debug "${1}"
                fi
            fi
        fi
    fi
}

# process the list of files and permissions for the filesystem
# $1 = action: create_tar, create_list
# $2 = fs list
# $3 = FS_OUTFILE
process_filesystem_list()
{
    ACTION="${1}"
    FILESYSTEM_LIST="${2}"
    FS_OUTFILE="${3}"

    echo "-> Creating archive $FS_OUTFILE containing the root file system"
    if ! [ -e "${FILESYSTEM_LIST}" ] ; then
        echo "   ERROR: the given list file \"${FILESYSTEM_LIST}\" does not exist"
        exit 1
    fi

    rm -f "${FS_OUTFILE}"
    rm -rf "${FS_TARGET_DIR}"
    mkdir "${FS_TARGET_DIR}"

    test -d $(dirname ${FS_OUTFILE}) || mkdir -p $(dirname ${FS_OUTFILE})
    OLDIFS=$IFS
    IFS=$'\n'
    for LINE in $(cat ${FILESYSTEM_LIST} | grep -F -v '#' | grep -Ev '^$' ) ; do
        TYPE=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f1 | sed 's/^ *//' | sed 's/ *$//' )
        DESTINATION_FILE=$(eval echo ${LINE} | tr -s ' ' | cut -s -d' ' -f2 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_1=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f3 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_2=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f4 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_3=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f5 | sed 's/^ *//' | sed 's/ *$//' )
        PARAM_4=$(echo ${LINE} | tr -s ' ' | cut -s -d' ' -f6 | sed 's/^ *//' | sed 's/ *$//' )

{'fraction_non_alphanumeric': 0.1732351135666053, 'fraction_numerical': 0.010067526089625536, 'mean_word_length': 1.8070296347346657, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2847855', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2695, 'n_words': 765}