<commit_before>#!/bin/bash

# Copyright 2015 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Discover all the ephemeral disks

block_devices=()

ephemeral_devices=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/ | grep ephemeral)
for ephemeral_device in $ephemeral_devices; do
  echo "Checking ephemeral device: ${ephemeral_device}"
  aws_device=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/${ephemeral_device})

  device_path=""
  if [ -b /dev/$aws_device ]; then
    device_path="/dev/$aws_device"
  else
    # Check for the xvd-style name
    xvd_style=$(echo $aws_device | sed "s/sd/xvd/")
    if [ -b /dev/$xvd_style ]; then
      device_path="/dev/$xvd_style"
    fi
  fi

  if [[ -z ${device_path} ]]; then
    echo "  Could not find disk: ${ephemeral_device}@${aws_device}"
  else
    echo "  Detected ephemeral disk: ${ephemeral_device}@${device_path}"
    block_devices+=(${device_path})
  fi
done

# These are set if we should move where docker/kubelet store data
# Note this gets set to the parent directory
move_docker=""
move_kubelet=""

apt-get update

docker_storage=${DOCKER_STORAGE:-aufs}

# Format the ephemeral disks
if [[ ${#block_devices[@]} == 0 ]]; then
  echo "No ephemeral block devices found; will use aufs on root"
  docker_storage="aufs"
else
  echo "Block devices: ${block_devices[@]}"

  if [[ ${docker_storage} == "btrfs" ]]; then
    apt-get install --yes btrfs-tools

    if [[ ${#block_devices[@]} == 1 ]]; then
      echo "One ephemeral block device found; formatting with btrfs"
      mkfs.btrfs -f ${block_devices[0]}
    else
      echo "Found multiple ephemeral block devices, formatting with btrfs as RAID-0"
      mkfs.btrfs -f --data raid0 ${block_devices[@]}
    fi
    mount -t btrfs ${block_devices[0]} /mnt

    mkdir -p /mnt/kubernetes

    move_docker="/mnt"
    move_kubelet="/mnt/kubernetes"
  elif [[ ${docker_storage} == "aufs-nolvm" ]]; then
    if [[ ${#block_devices[@]} != 1 ]]; then
      echo "aufs-nolvm selected, but multiple ephemeral devices were found; only the first will be available"
    fi

    /bin/umount ${block_devices[0]}
    mkfs -t ext4 ${block_devices[0]}
    mount -t ext4 ${block_devices[0]} /mnt

    mkdir -p /mnt/kubernetes

    move_docker="/mnt"
    move_kubelet="/mnt/kubernetes"
  elif [[ ${docker_storage} == "devicemapper" || ${docker_storage} == "aufs" ]]; then
    # We always use LVM, even with one device
    # In devicemapper mode, Docker can use LVM directly
    # Also, fewer code paths are good
    echo "Using LVM2 and ext4"
    apt-get install --yes lvm2

    # Don't output spurious "File descriptor X leaked on vgcreate invocation."
    # Known bug: e.g. Ubuntu #591823
    export LVM_SUPPRESS_FD_WARNINGS=1

    for block_device in ${block_devices}; do
      /bin/umount ${block_device}
      pvcreate ${block_device}
    done
    vgcreate vg-ephemeral ${block_devices[@]}

    if [[ ${docker_storage} == "devicemapper" ]]; then
      # devicemapper thin provisioning, managed by docker
      # This is the best option, but it is sadly broken on most distros
      # Bug: https://github.com/docker/docker/issues/4036

      # 95% goes to the docker thin-pool
      lvcreate -l 95%VG --thinpool docker-thinpool vg-ephemeral

      DOCKER_OPTS="${DOCKER_OPTS} --storage-opt dm.thinpooldev=/dev/mapper/vg--ephemeral-docker--thinpool"
      # Note that we don't move docker; docker goes direct to the thinpool
    else
      # aufs

      # Create a docker lv, use docker on it
      # 95% goes to the docker thin-pool
      lvcreate -l 95%VG --thinpool docker-thinpool vg-ephemeral

      THINPOOL_SIZE=$(lvs vg-ephemeral/docker-thinpool -o LV_SIZE --noheadings --units M --nosuffix)
      lvcreate -V${THINPOOL_SIZE}M -T vg-ephemeral/docker-thinpool -n docker

      mkfs -t ext4 /dev/vg-ephemeral/docker
      mkdir -p /mnt/docker
      mount -t ext4 /dev/vg-ephemeral/docker /mnt/docker
      move_docker="/mnt"
    fi

    # Remaining 5% is for kubernetes data
    # TODO: Should this be a thin pool?  e.g. would we ever want to snapshot this data?
    lvcreate -l 100%FREE -n kubernetes vg-ephemeral
    mkfs -t ext4 /dev/vg-ephemeral/kubernetes
    mkdir -p /mnt/kubernetes
    mount -t ext4 /dev/vg-ephemeral/kubernetes /mnt/kubernetes
    move_kubelet="/mnt/kubernetes"
  else<commit_msg>AWS: Don't thin provision LVM volume on wheezy<commit_after>#!/bin/bash

# Copyright 2015 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Discover all the ephemeral disks

block_devices=()

ephemeral_devices=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/ | grep ephemeral)
for ephemeral_device in $ephemeral_devices; do
  echo "Checking ephemeral device: ${ephemeral_device}"
  aws_device=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/${ephemeral_device})

  device_path=""
  if [ -b /dev/$aws_device ]; then
    device_path="/dev/$aws_device"
  else
    # Check for the xvd-style name
    xvd_style=$(echo $aws_device | sed "s/sd/xvd/")
    if [ -b /dev/$xvd_style ]; then
      device_path="/dev/$xvd_style"
    fi
  fi

  if [[ -z ${device_path} ]]; then
    echo "  Could not find disk: ${ephemeral_device}@${aws_device}"
  else
    echo "  Detected ephemeral disk: ${ephemeral_device}@${device_path}"
    block_devices+=(${device_path})
  fi
done

# These are set if we should move where docker/kubelet store data
# Note this gets set to the parent directory
move_docker=""
move_kubelet=""

apt-get update

docker_storage=${DOCKER_STORAGE:-aufs}

# Format the ephemeral disks
if [[ ${#block_devices[@]} == 0 ]]; then
  echo "No ephemeral block devices found; will use aufs on root"
  docker_storage="aufs"
else
  echo "Block devices: ${block_devices[@]}"

  if [[ ${docker_storage} == "btrfs" ]]; then
    apt-get install --yes btrfs-tools

    if [[ ${#block_devices[@]} == 1 ]]; then
      echo "One ephemeral block device found; formatting with btrfs"
      mkfs.btrfs -f ${block_devices[0]}
    else
      echo "Found multiple ephemeral block devices, formatting with btrfs as RAID-0"
      mkfs.btrfs -f --data raid0 ${block_devices[@]}
    fi
    mount -t btrfs ${block_devices[0]} /mnt

    mkdir -p /mnt/kubernetes

    move_docker="/mnt"
    move_kubelet="/mnt/kubernetes"
  elif [[ ${docker_storage} == "aufs-nolvm" ]]; then
    if [[ ${#block_devices[@]} != 1 ]]; then
      echo "aufs-nolvm selected, but multiple ephemeral devices were found; only the first will be available"
    fi

    /bin/umount ${block_devices[0]}
    mkfs -t ext4 ${block_devices[0]}
    mount -t ext4 ${block_devices[0]} /mnt

    mkdir -p /mnt/kubernetes

    move_docker="/mnt"
    move_kubelet="/mnt/kubernetes"
  elif [[ ${docker_storage} == "devicemapper" || ${docker_storage} == "aufs" ]]; then
    # We always use LVM, even with one device
    # In devicemapper mode, Docker can use LVM directly
    # Also, fewer code paths are good
    echo "Using LVM2 and ext4"
    apt-get install --yes lvm2

    # Don't output spurious "File descriptor X leaked on vgcreate invocation."
    # Known bug: e.g. Ubuntu #591823
    export LVM_SUPPRESS_FD_WARNINGS=1

    for block_device in ${block_devices}; do
      /bin/umount ${block_device}
      pvcreate ${block_device}
    done
    vgcreate vg-ephemeral ${block_devices[@]}

    if [[ ${docker_storage} == "devicemapper" ]]; then
      # devicemapper thin provisioning, managed by docker
      # This is the best option, but it is sadly broken on most distros
      # Bug: https://github.com/docker/docker/issues/4036

      # 95% goes to the docker thin-pool
      lvcreate -l 95%VG --thinpool docker-thinpool vg-ephemeral

      DOCKER_OPTS="${DOCKER_OPTS} --storage-opt dm.thinpooldev=/dev/mapper/vg--ephemeral-docker--thinpool"
      # Note that we don't move docker; docker goes direct to the thinpool
    else
      # aufs

      # Create a docker lv, use docker on it
      # 95% goes to the docker thin-pool
      release=`lsb_release -c -s`
      if [[ "${release}" != "wheezy" ]] ; then
        lvcreate -l 95%VG --thinpool docker-thinpool vg-ephemeral

        THINPOOL_SIZE=$(lvs vg-ephemeral/docker-thinpool -o LV_SIZE --noheadings --units M --nosuffix)
        lvcreate -V${THINPOOL_SIZE}M -T vg-ephemeral/docker-thinpool -n docker
      else
        # Thin provisioning not supported by Wheezy
        echo "Detected wheezy; won't use LVM thin provisioning"
        lvcreate -l 95%VG -n docker vg-ephemeral
      fi

      mkfs -t ext4 /dev/vg-ephemeral/docker
      mkdir -p /mnt/docker
      mount -t ext4 /dev/vg-ephemeral/docker /mnt/docker
      move_docker="/mnt"
    fi

    # Remaining 5% is for kubernetes data
    # TODO: Should this be a thin pool?  e.g. would we ever want to snapshot this data?
    lvcreate -l 100%FREE -n kubernetes vg-ephemeral
    mkfs -t ext4 /dev/vg-ephemeral/kubernetes
    mkdir -p /mnt/kubernetes
    mount -t ext4 /dev/vg-ephemeral/kubernetes /mnt/kubernetes
    move_kubelet="/mnt/kubernetes"
  else
{'fraction_non_alphanumeric': 0.11542642553612018, 'fraction_numerical': 0.01798596699278585, 'mean_word_length': 3.19394944053046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3088853', 'n_tokens_mistral': 3680, 'n_tokens_neox': 3308, 'n_words': 1045}