<commit_before>#!/bin/bash
set -euxo pipefail

# Mount instance storage if we can
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html

# Move docker root to the ephemeral device
if [[ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" != "true" ]] ; then
  echo "Skipping mounting instance storage"
  exit 0
fi

devicemount=/ephemeral
devices=($(sudo nvme list | grep "Amazon EC2 NVMe Instance Storage"| cut -f1 -d' '))

if [ -z "${devices[@]}" ]
then
  echo "No Instance Storage NVMe drives to mount" >&2
  exit 0
fi

if [[ "${#devices[@]}" -eq 1 ]] ; then
  logicalname="${devices[0]}"
elif [[ "${#devices[@]}" -gt 1 ]] ; then
  logicalname=/dev/md0

  mdadm \
    --create "$logicalname" \
    --level=0 \
    -c256 \
    --raid-devices="${#devices[@]}" "${devices[@]}"

  echo "DEVICE ${devices[*]}" > /etc/mdadm.conf

  mdadm --detail --scan >> /etc/mdadm.conf
  blockdev --setra 65536 "$logicalname"
fi

# Make an ext4 file system, [-F]orce creation
mkfs.ext4 -F -E nodiscard "$logicalname" > /dev/null

mkdir -p "$devicemount"
mount -t ext4 -o noatime "$logicalname" "$devicemount"

if [ ! -f /etc/fstab.backup ]; then
  cp -rP /etc/fstab /etc/fstab.backup
  echo "$logicalname $devicemount    ext4  defaults  0 0" >> /etc/fstab
fi
<commit_msg>Mount the device fs or raid fs at /mnt/ephemeral<commit_after>#!/bin/bash
set -euxo pipefail

# Mount instance storage if we can
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html

# Move docker root to the ephemeral device
if [[ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" != "true" ]] ; then
  echo "Skipping mounting instance storage"
  exit 0
fi

devices=($(sudo nvme list | grep "Amazon EC2 NVMe Instance Storage"| cut -f1 -d' '))

if [ -z "${devices[@]}" ]
then
  echo "No Instance Storage NVMe drives to mount" >&2
  exit 0
fi

if [[ "${#devices[@]}" -eq 1 ]] ; then
  logicalname="${devices[0]}"
elif [[ "${#devices[@]}" -gt 1 ]] ; then
  logicalname=/dev/md0

  mdadm \
    --create "$logicalname" \
    --level=0 \
    -c256 \
    --raid-devices="${#devices[@]}" "${devices[@]}"

  echo "DEVICE ${devices[*]}" > /etc/mdadm.conf

  mdadm --detail --scan >> /etc/mdadm.conf
  blockdev --setra 65536 "$logicalname"
fi

# Make an ext4 file system, [-F]orce creation
mkfs.ext4 -F -E nodiscard "$logicalname" > /dev/null

devicemount=/mnt/ephemeral
mkdir -p "$devicemount"
mount -t ext4 -o noatime "$logicalname" "$devicemount"

if [ ! -f /etc/fstab.backup ]; then
  cp -rP /etc/fstab /etc/fstab.backup
  echo "$logicalname $devicemount    ext4  defaults  0 0" >> /etc/fstab
fi

{'fraction_non_alphanumeric': 0.17190531625921615, 'fraction_numerical': 0.019402405898331393, 'mean_word_length': 3.4602076124567476, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1685856', 'n_tokens_mistral': 1027, 'n_tokens_neox': 961, 'n_words': 270}