<commit_before>#!/bin/bash
# set -eo pipefail

PRE_SCRIPTS_DIR=/opt/skiff/scripts/mount-all.pre.d
# Run any additional pre setup scripts.
# We source these to allow overriding the above variables.
for i in ${PRE_SCRIPTS_DIR}/*.sh ; do
    if [ -r $i ]; then
        source $i
    fi
done

if [ -n "$PERSIST_RESIZE_DEVICE" ]; then
    PERSIST_DEVICE=$PERSIST_RESIZE_DEVICE
fi

if [ -n "$PERSIST_DEVICE" ]; then
    PERSIST_DEV=$(blkid | grep ${PERSIST_DEVICE}: | head -n1 | cut -d: -f1)
fi
if [ -z "$PERSIST_DEV" ]; then
    PERSIST_DEV=/dev/disk/by-label/persist
fi
udevadm settle || true
if [ ! -b "$PERSIST_DEV" ]; then
    echo "Cannot find persist device, will retry in a moment..."
    sleep 2
fi
if [ ! -b "$PERSIST_DEV" ]; then
    echo "Cannot find persist device, skipping check."
    echo "PERSIST_DEVICE=${PERSIST_DEVICE}"
    echo "PERSIST_DEV=${PERSIST_DEV}"
    exit 0
fi
PERSIST_DEV=$(readlink -f $PERSIST_DEV)

echo "Found persist partition at ${PERSIST_DEV}, performing filesystem check if necessary..."
FSCKFIX=yes fsck -y $PERSIST_DEV

PARTNAME=$(basename $PERSIST_DEV)
PARTLINE=$(cat /proc/partitions | grep $PARTNAME)
DEVMAJOR=$(echo $PARTLINE | awk '{print $1}')
DEVMINOR=$(echo $PARTLINE | awk '{print $2}')
DEVNAME=$(cat /proc/partitions | grep " $DEVMAJOR " | head -n1 | awk '{print $4}')
DEVPATH=/dev/$DEVNAME
echo "Disk partition detected ${DEVPATH}"
if [ ! -b $DEVPATH ]; then
    echo "Cannot find disk at $DEVPATH - continuing"
    exit 0
fi
disk=$DEVPATH
echo "Disk detected at ${disk}."
part_num=$DEVMINOR
echo "Partition number detected as $part_num"
disk_part=$PERSIST_DEV
p2_start=$(fdisk -l $disk -o "DEVICE,START" | grep -w $disk_part | awk '{print $2}' | tr -d '[[:space:]]')
echo "Partition start: $p2_start"
p2_end=$(fdisk -l $disk -o "DEVICE,END" | grep -w $disk_part | awk '{print $2}' | tr -d '[[:space:]]')
echo "Partition end: $p2_end"
disk_size=$(blockdev --getsize $disk)
echo "Disk size: $disk_size"
<commit_msg>resize-persist: run e2fsck fix before resize<commit_after>#!/bin/bash
# set -eo pipefail

PRE_SCRIPTS_DIR=/opt/skiff/scripts/mount-all.pre.d
# Run any additional pre setup scripts.
# We source these to allow overriding the above variables.
for i in ${PRE_SCRIPTS_DIR}/*.sh ; do
    if [ -r $i ]; then
        source $i
    fi
done

if [ -n "$PERSIST_RESIZE_DEVICE" ]; then
    PERSIST_DEVICE=$PERSIST_RESIZE_DEVICE
fi

if [ -n "$PERSIST_DEVICE" ]; then
    PERSIST_DEV=$(blkid | grep ${PERSIST_DEVICE}: | head -n1 | cut -d: -f1)
fi
if [ -z "$PERSIST_DEV" ]; then
    PERSIST_DEV=/dev/disk/by-label/persist
fi
udevadm settle || true
if [ ! -b "$PERSIST_DEV" ]; then
    echo "Cannot find persist device, will retry in a moment..."
    sleep 2
fi
if [ ! -b "$PERSIST_DEV" ]; then
    echo "Cannot find persist device, skipping check."
    echo "PERSIST_DEVICE=${PERSIST_DEVICE}"
    echo "PERSIST_DEV=${PERSIST_DEV}"
    exit 0
fi
PERSIST_DEV=$(readlink -f $PERSIST_DEV)

echo "Found persist partition at ${PERSIST_DEV}, performing filesystem check if necessary..."
FSCKFIX=yes fsck -y $PERSIST_DEV || true
FSCKFIX=yes e2fsck -y -f $PERSIST_DEV || true

PARTNAME=$(basename $PERSIST_DEV)
PARTLINE=$(cat /proc/partitions | grep $PARTNAME)
DEVMAJOR=$(echo $PARTLINE | awk '{print $1}')
DEVMINOR=$(echo $PARTLINE | awk '{print $2}')
DEVNAME=$(cat /proc/partitions | grep " $DEVMAJOR " | head -n1 | awk '{print $4}')
DEVPATH=/dev/$DEVNAME
echo "Disk partition detected ${DEVPATH}"
if [ ! -b $DEVPATH ]; then
    echo "Cannot find disk at $DEVPATH - continuing"
    exit 0
fi
disk=$DEVPATH
echo "Disk detected at ${disk}."
part_num=$DEVMINOR
echo "Partition number detected as $part_num"
disk_part=$PERSIST_DEV
p2_start=$(fdisk -l $disk -o "DEVICE,START" | grep -w $disk_part | awk '{print $2}' | tr -d '[[:space:]]')
echo "Partition start: $p2_start"
p2_end=$(fdisk -l $disk -o "DEVICE,END" | grep -w $disk_part | awk '{print $2}' | tr -d '[[:space:]]')
echo "Partition end: $p2_end"
disk_size=$(blockdev --getsize $disk)
echo "Disk size: $disk_size"

{'fraction_non_alphanumeric': 0.1523096129837703, 'fraction_numerical': 0.007990012484394507, 'mean_word_length': 4.1757105943152455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '937454', 'n_tokens_mistral': 1510, 'n_tokens_neox': 1481, 'n_words': 450}