<commit_before>#!/bin/bash

# Do not add {} around variables without it: it is on purpose so that replace_vars do not replace them.

# Mount volumes
mount -a

set -e

cd `dirname $0`

# Install Docker
docker --version || INSTALL_DOCKER=1
if [ "$INSTALL_DOCKER" = "1" ]; then
    apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys <KEY>
    echo "deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -cs) main" | tee /etc/apt/sources.list.d/docker.list -
    apt-get update
    apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual
    apt-get install -y docker-engine
    service docker start || echo 'Already running'
fi

# Search for unmounted volume
device_name=/dev/xvdb
mount_point=/mnt
if [ -e $device_name ] && [ `df | grep $device_name | wc -l` == "0" ] && [ ! -e $mount_point ]; then
    mkfs -t ext4 $device_name
    mkdir -p $mount_point
    mount $device_name $mount_point

    if [ `cat /etc/fstab | grep $device_name | wc -l` == "0" ]; then
        echo "$device_name       $mount_point   ext4    defaults,nofail,nobootwait        0       2"  >> /etc/fstab
    fi
fi

# Install NVIDIA drivers
if [ "${DOCKER_CMD}" = "nvidia-docker" ]; then
    if [ ! `which nvidia-smi` ]; then
        apt-get update
        apt-get install nvidia-375
    fi
    # Make sure modules are loaded
    modprobe nvidia
    modprobe nvidia_uvm
    if [ ! `which nvidia-docker` ]; then
        apt-get update
        apt-get install nvidia-modprobe
        wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb
        sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb
    fi
fi

# Copy docker credentials
cp -r .docker* $HOME/ || :

# Launch links if not launched
${LAUNCH_LINK}

# Pull the base image and build
docker pull ${IMAGE_NAME}

RUN_COMMAND="${DOCKER_CMD} run ${DOCKER_OPTS} -v /var/log:/var/log"
# Options to share docker (if the hooks wants to launch another container)
DOCKER_SHARE_OPTS="-v /var/run/docker.sock:/var/run/docker.sock -v $(which docker):/usr/bin/docker -v /usr/lib/x86_64-linux-gnu/libltdl.so.7:/usr/lib/x86_64-linux-gnu/libltdl.so.7"
RUN_COMMAND_HOOKS="$RUN_COMMAND --rm $DOCKER_SHARE_OPTS -t -i ${IMAGE_NAME}"

# Run pre hooks (deprecated)
if [ ! -z "${PRE_DEPLOY_HOOKS}" ]; then
    echo "Running pre-deploy script ${PRE_DEPLOY_HOOKS}"
    $RUN_COMMAND_HOOKS ${PRE_DEPLOY_HOOKS}
fi
<commit_msg> SSH deployment: Install nvidia-docker v2 instead of v1. (#73)<commit_after>#!/bin/bash

# Do not add {} around variables without it: it is on purpose so that replace_vars do not replace them.

# Mount volumes
mount -a

set -e

cd `dirname $0`

# Install Docker
docker --version || INSTALL_DOCKER=1
if [ "$INSTALL_DOCKER" = "1" ]; then
    apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys <KEY>
    echo "deb https://apt.dockerproject.org/repo ubuntu-$(lsb_release -cs) main" | tee /etc/apt/sources.list.d/docker.list -
    apt-get update
    apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual
    apt-get install -y docker-engine
    service docker start || echo 'Already running'
fi

# Search for unmounted volume
device_name=/dev/xvdb
mount_point=/mnt
if [ -e $device_name ] && [ `df | grep $device_name | wc -l` == "0" ] && [ ! -e $mount_point ]; then
    mkfs -t ext4 $device_name
    mkdir -p $mount_point
    mount $device_name $mount_point

    if [ `cat /etc/fstab | grep $device_name | wc -l` == "0" ]; then
        echo "$device_name       $mount_point   ext4    defaults,nofail,nobootwait        0       2"  >> /etc/fstab
    fi
fi

# Install NVIDIA drivers
if [ "${DOCKER_CMD}" = "nvidia-docker" ]; then
    if [ ! `which nvidia-smi` ]; then
        apt-get update
        apt-get install nvidia-384
    fi
    # Make sure modules are loaded
    modprobe nvidia
    modprobe nvidia_uvm
    if [ ! `which nvidia-docker` ]; then
        # install nvidia-docker v2
        curl -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add -
        tee /etc/apt/sources.list.d/nvidia-docker.list <<< \
"deb https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 /
deb https://nvidia.github.io/nvidia-container-runtime/ubuntu16.04/amd64 /
deb https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64 /"
        apt-get update
        apt-get install nvidia-docker2
        systemctl reload docker
    fi
fi

# Copy docker credentials
cp -r .docker* $HOME/ || :

# Launch links if not launched
${LAUNCH_LINK}

# Pull the base image and build
docker pull ${IMAGE_NAME}

RUN_COMMAND="${DOCKER_CMD} run ${DOCKER_OPTS} -v /var/log:/var/log"
# Options to share docker (if the hooks wants to launch another container)
DOCKER_SHARE_OPTS="-v /var/run/docker.sock:/var/run/docker.sock -v $(which docker):/usr/bin/docker -v /usr/lib/x86_64-linux-gnu/libltdl.so.7:/usr/lib/x86_64-linux-gnu/libltdl.so.7"
RUN_COMMAND_HOOKS="$RUN_COMMAND --rm $DOCKER_SHARE_OPTS -t -i ${IMAGE_NAME}"

# Run pre hooks (deprecated)
if [ ! -z "${PRE_DEPLOY_HOOKS}" ]; then
    echo "Running pre-deploy script ${PRE_DEPLOY_HOOKS}"
    $RUN_COMMAND_HOOKS ${PRE_DEPLOY_HOOKS}
fi

{'fraction_non_alphanumeric': 0.13497766556612933, 'fraction_numerical': 0.01650805981744028, 'mean_word_length': 3.0711462450592886, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '824776', 'n_tokens_mistral': 2032, 'n_tokens_neox': 1858, 'n_words': 482}