<commit_before>#!/bin/bash -e
set -x
# This script should be run only inside of a Docker container
if [ ! -f /.dockerenv ]; then
  echo "ERROR: script works only in a Docker container!"
  exit 1
fi

# get versions for software that needs to be installed
source /workspace/versions.config

### setting up some important variables to control the build process

# place to store our created sd-image file
BUILD_RESULT_PATH="/workspace"

# place to build our sd-image
BUILD_PATH="/build"

# Show TRAVSI_TAG in travis builds
echo TRAVIS_TAG="${TRAVIS_TAG}"

HYPRIOT_IMAGE_ZIP="hypriotos-rpi-${HYPRIOT_OS_VERSION}.img.zip"
HYPRIOT_IMAGE_PATH_ZIP="${BUILD_RESULT_PATH}/${HYPRIOT_IMAGE_ZIP}"

HYPRIOT_IMAGE="hypriotos-rpi-${HYPRIOT_OS_VERSION}.img"
HYPRIOT_IMAGE_PATH="${BUILD_RESULT_PATH}/${HYPRIOT_IMAGE}"

# create build directory for assembling our image filesystem
rm -rf ${BUILD_PATH}
mkdir ${BUILD_PATH}

# download HypriotOS
if [ ! -f "${HYPRIOT_IMAGE_PATH}" ]; then
  wget -q -O "${HYPRIOT_IMAGE_PATH_ZIP}" "https://github.com/hypriot/image-builder-rpi/releases/download/${HYPRIOT_OS_VERSION}/${HYPRIOT_IMAGE_ZIP}"

  # verify checksum of our root filesystem
  echo "${HYPRIOT_IMAGE_CHECKSUM} ${HYPRIOT_IMAGE_PATH_ZIP}" | sha256sum -c -

  # extract HypriotOS image
  unzip -p "${HYPRIOT_IMAGE_PATH_ZIP}" > "${HYPRIOT_IMAGE_PATH}"

  rm "${HYPRIOT_IMAGE_PATH_ZIP}"
fi


# extract parts of image

# delete image zip


# # register qemu-arm with binfmt
# # to ensure that binaries we use in the chroot
# # are executed via qemu-arm
# update-binfmts --enable qemu-arm

# # set up mount points for the pseudo filesystems
# mkdir -p ${BUILD_PATH}/{proc,sys,dev/pts}

# mount -o bind /dev ${BUILD_PATH}/dev
# mount -o bind /dev/pts ${BUILD_PATH}/dev/pts
# mount -t proc none ${BUILD_PATH}/proc
# mount -t sysfs none ${BUILD_PATH}/sys

# # modify/add image files directly
# # e.g. root partition resize script
# cp -R /builder/files/* ${BUILD_PATH}/

# # make our build directory the current root
# # and install the Rasberry Pi firmware, kernel packages,
# # docker tools and some customizations
# chroot ${BUILD_PATH} /bin/bash < /builder/chroot-script.sh

# # unmount pseudo filesystems
# umount -l ${BUILD_PATH}/dev/pts
# umount -l ${BUILD_PATH}/dev<commit_msg>Pull out parts of the image<commit_after>#!/bin/bash -e
set -x
# This script should be run only inside of a Docker container
if [ ! -f /.dockerenv ]; then
  echo "ERROR: script works only in a Docker container!"
  exit 1
fi

# get versions for software that needs to be installed
source /workspace/versions.config

### setting up some important variables to control the build process

# place to store our created sd-image file
BUILD_RESULT_PATH="/workspace"

# place to build our sd-image
BUILD_PATH="/build"

# Show TRAVSI_TAG in travis builds
echo TRAVIS_TAG="${TRAVIS_TAG}"

HYPRIOT_IMAGE_ZIP="hypriotos-rpi-${HYPRIOT_OS_VERSION}.img.zip"
HYPRIOT_IMAGE_PATH_ZIP="${BUILD_RESULT_PATH}/${HYPRIOT_IMAGE_ZIP}"

HYPRIOT_IMAGE="hypriotos-rpi-${HYPRIOT_OS_VERSION}.img"
HYPRIOT_IMAGE_PATH="${BUILD_RESULT_PATH}/${HYPRIOT_IMAGE}"

# create build directory for assembling our image filesystem
rm -rf ${BUILD_PATH}
mkdir ${BUILD_PATH}

# download HypriotOS
if [ ! -f "${HYPRIOT_IMAGE_PATH}" ]; then
  wget -q -O "${HYPRIOT_IMAGE_PATH_ZIP}" "https://github.com/hypriot/image-builder-rpi/releases/download/${HYPRIOT_OS_VERSION}/${HYPRIOT_IMAGE_ZIP}"

  # verify checksum of our root filesystem
  echo "${HYPRIOT_IMAGE_CHECKSUM} ${HYPRIOT_IMAGE_PATH_ZIP}" | sha256sum -c -

  # extract HypriotOS image
  unzip -p "${HYPRIOT_IMAGE_PATH_ZIP}" > "${HYPRIOT_IMAGE_PATH}"

  rm "${HYPRIOT_IMAGE_PATH_ZIP}"
fi

# extract parts of image
guestfish -a "/${HYPRIOT_IMAGE_PATH}"<<_EOF_
  run
  #import filesystem content
  mount /dev/sda2 /
  tar-out / /image_with_kernel_root.tar.gz compress:gzip
  mount /dev/sda1 /boot
  tar-out /boot /image_with_kernel_boot.tar.gz compress:gzip
_EOF_

# untar file system to BUILD_PATH
tar -zxf image_with_kernel_root.tar.gz -C ${BUILD_PATH}
tar -zxf image_with_kernel_boot.tar.gz -C ${BUILD_PATH}/boot/

# register qemu-arm with binfmt
# to ensure that binaries we use in the chroot
# are executed via qemu-arm
update-binfmts --enable qemu-arm

# set up mount points for the pseudo filesystems
mkdir -p ${BUILD_PATH}/{proc,sys,dev/pts}

mount -o bind /dev ${BUILD_PATH}/dev
mount -o bind /dev/pts ${BUILD_PATH}/dev/pts
mount -t proc none ${BUILD_PATH}/proc
mount -t sysfs none ${BUILD_PATH}/sys

# # modify/add image files directly
# # e.g. root partition resize script
# cp -R /builder/files/* ${BUILD_PATH}/

# # make our build directory the current root
# # and install the Rasberry Pi firmware, kernel packages,
# # docker tools and some customizations
# chroot ${BUILD_PATH} /bin/bash < /builder/chroot-script.sh

# # unmount pseudo filesystems
# umount -l ${BUILD_PATH}/dev/pts
# umount -l ${BUILD_PATH}/dev
{'fraction_non_alphanumeric': 0.11694949907994275, 'fraction_numerical': 0.0020445716622367613, 'mean_word_length': 4.688372093023256, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7509490', 'n_tokens_mistral': 1743, 'n_tokens_neox': 1712, 'n_words': 488}