<commit_before>#!/bin/bash
set -e

function zap_device {
  local device_match_string='/dev/([hsv]d[a-z]{1,2}|cciss/c[0-9]d[0-9]p|nvme[0-9]n[0-9]p){1,2}'

  if [[ -z ${OSD_DEVICE} ]]; then
    log "Please provide device(s) to zap!"
    log "ie: '-e OSD_DEVICE=/dev/sdb' or '-e OSD_DEVICE=/dev/sdb,/dev/sdc'"
    exit 1
  fi

  # testing all the devices first so we just don't do anything if one device is wrong
  for device in $(echo ${OSD_DEVICE} | tr "," " "); do
    if ! file -s $device &> /dev/null; then
      log "Provided device $device does not exist."
      exit 1
    fi
    # if the disk passed is a raw device AND the boot system disk
    if echo $device | egrep -sq ${device_match_string} && parted -s $(echo $device | egrep -o ${device_match_string}) print | grep -sq boot; then
      log "Looks like $device has a boot partition,"
      log "if you want to delete specific partitions point to the partition instead of the raw device"
      log "Do not use your system disk!"
      exit 1
    fi
  done

  # look for Ceph encrypted partitions
  ceph_dm=$(blkid -t TYPE="crypto_LUKS" ${OSD_DEVICE}* -o value -s PARTUUID || true)
  if [[ -n $ceph_dm ]]; then
    for dm_uuid in $ceph_dm; do
      dmsetup --verbose --force wipe_table $dm_uuid || true
      dmsetup --verbose --force remove $dm_uuid || true
      # erase all keyslots (remove encryption key)
      cryptsetup --verbose --batch-mode erase /dev/disk/by-partuuid/$dm_uuid
      payload_offset=$(cryptsetup luksDump /dev/disk/by-partuuid/$dm_uuid | awk '/Payload offset:/ { print $3 }')
      # remove LUKS header
      dd if=/dev/zero of=/dev/disk/by-partuuid/$dm_uuid bs=512 count=$payload_offset oflag=direct
    done
  fi

  for device in $(echo ${OSD_DEVICE} | tr "," " "); do
    raw_device=$(echo $device | egrep -o ${device_match_string})
    if echo $device | egrep -sq ${device_match_string}; then
      log "Zapping the entire device $device"
      sgdisk --zap-all --clear --mbrtogpt -g -- $device
    else
      # get the desired partition number(s)
      partition_nb=$(echo $device | egrep -o '[0-9]{1,2}$')
      log "Zapping partition $device"
      sgdisk --delete $partition_nb $raw_device
    fi
    log "Executing partprobe on $raw_device"<commit_msg>zap_device: 'Payload offset' unit is sector size<commit_after>#!/bin/bash
set -e

function zap_device {
  local device_match_string='/dev/([hsv]d[a-z]{1,2}|cciss/c[0-9]d[0-9]p|nvme[0-9]n[0-9]p){1,2}'

  if [[ -z ${OSD_DEVICE} ]]; then
    log "Please provide device(s) to zap!"
    log "ie: '-e OSD_DEVICE=/dev/sdb' or '-e OSD_DEVICE=/dev/sdb,/dev/sdc'"
    exit 1
  fi

  # testing all the devices first so we just don't do anything if one device is wrong
  for device in $(echo ${OSD_DEVICE} | tr "," " "); do
    if ! file -s $device &> /dev/null; then
      log "Provided device $device does not exist."
      exit 1
    fi
    # if the disk passed is a raw device AND the boot system disk
    if echo $device | egrep -sq ${device_match_string} && parted -s $(echo $device | egrep -o ${device_match_string}) print | grep -sq boot; then
      log "Looks like $device has a boot partition,"
      log "if you want to delete specific partitions point to the partition instead of the raw device"
      log "Do not use your system disk!"
      exit 1
    fi
  done

  # look for Ceph encrypted partitions
  ceph_dm=$(blkid -t TYPE="crypto_LUKS" ${OSD_DEVICE}* -o value -s PARTUUID || true)
  if [[ -n $ceph_dm ]]; then
    for dm_uuid in $ceph_dm; do
      dmsetup --verbose --force wipe_table $dm_uuid || true
      dmsetup --verbose --force remove $dm_uuid || true
      # erase all keyslots (remove encryption key)
      cryptsetup --verbose --batch-mode erase /dev/disk/by-partuuid/$dm_uuid
      payload_offset=$(cryptsetup luksDump /dev/disk/by-partuuid/$dm_uuid | awk '/Payload offset:/ { print $3 }')
      phys_sector_size=$(blockdev --getpbsz /dev/disk/by-partuuid/$dm_uuid)
      if ! is_integer "$phys_sector_size"; then
        # If the sector size isn't a number, let's default to 512
        phys_sector_size=512
      fi
      # remove LUKS header
      dd if=/dev/zero of=/dev/disk/by-partuuid/$dm_uuid bs=$phys_sector_size count=$payload_offset oflag=direct
    done
  fi

  for device in $(echo ${OSD_DEVICE} | tr "," " "); do
    raw_device=$(echo $device | egrep -o ${device_match_string})
    if echo $device | egrep -sq ${device_match_string}; then
      log "Zapping the entire device $device"
      sgdisk --zap-all --clear --mbrtogpt -g -- $device
    else
      # get the desired partition number(s)
      partition_nb=$(echo $device | egrep -o '[0-9]{1,2}$')
      log "Zapping partition $device"
      sgdisk --delete $partition_nb $raw_device
    fi
    log "Executing partprobe on $raw_device"
{'fraction_non_alphanumeric': 0.12832879010274692, 'fraction_numerical': 0.010274690710840848, 'mean_word_length': 2.948675496688742, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3829900', 'n_tokens_mistral': 1667, 'n_tokens_neox': 1599, 'n_words': 533}