<commit_before>#!/bin/bash

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

set -euxo pipefail

exec 1>&2

sudo apt-get update && sudo apt-get -y upgrade

sudo apt-get install -y csh wget curl vim git realpath tree htop libsnappy1v5 \
    lxc lvm2 xfsprogs pssh

pushd ~/var/yarn-ec2 > /dev/null

for vm in `sudo lxc-ls` ; do
    sudo lxc-stop -k -n $vm || :
    sudo lxc-destroy -f -n $vm
    sleep 0.1
done

sudo service lxc stop
sudo service lxc-net stop
sudo rm -f /var/lib/misc/dnsmasq.lxcbr0.leases
sudo killall -9 java || :
sleep 0.1

sudo rm -rf /tmp/Jetty*
sudo rm -rf /tmp/hadoop*
sudo rm -rf /tmp/hd*
sudo rm -rf /tmp/y*

sudo mkdir -p /opt/tarfiles
sudo chmod a+rx /opt/tarfiles
sudo rm -rf /opt/hadoop-*
sudo rm -rf /opt/jdk*

HADOOP_TGZ=hadoop-2.2.0.tar.gz
HADOOP_URL=https://s3.amazonaws.com/ubuntu-ursus-packages/$HADOOP_TGZ
[ ! -e /opt/tarfiles/$HADOOP_TGZ -o ! -s /opt/tarfiles/$HADOOP_TGZ ] && \
sudo wget --no-check-certificate $HADOOP_URL -O /opt/tarfiles/$HADOOP_TGZ
sudo tar xzf /opt/tarfiles/$HADOOP_TGZ -C /opt
sudo chown -R root:root /opt/hadoop-2.2.0
sudo umount -l /usr/local/hd || :
sudo mkdir -p /usr/local/hd
sudo mount --bind -o ro /opt/hadoop-2.2.0 /usr/local/hd
TAPACK_TGZ=advcc-sp1.tar.gz
TAPACK_URL=https://s3.amazonaws.com/ubuntu-ursus-packages/$TAPACK_TGZ
[ ! -e /opt/tarfiles/$TAPACK_TGZ -o ! -s /opt/tarfiles/$TAPACK_TGZ ] && \
    sudo wget --no-check-certificate $TAPACK_URL -O /opt/tarfiles/$TAPACK_TGZ
sudo tar xzf /opt/tarfiles/$TAPACK_TGZ -C /opt
sudo chown -R root:root /opt/advcc-sp1
sudo cp /opt/advcc-sp1/jobexec/* /opt/hadoop-2.2.0/
sudo mv /opt/hadoop-2.2.0/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-2.2.0.jar \
    /opt/hadoop-2.2.0/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-2.2.0.jar.origin
sudo cp /opt/advcc-sp1/hadoop-yarn-server-resourcemanager-2.2.0.jar /opt/hadoop-2.2.0/share/hadoop/yarn/
sudo cp /opt/advcc-sp1/hadoop-yarn-applications-mpirunner-2.2.0.jar /opt/hadoop-2.2.0/
sudo cp /opt/advcc-sp1/hadoop-yarn-applications-gpu-2.2.0.jar /opt/hadoop-2.2.0/

SUNJDK_TGZ=jdk-8u121-linux-x64.tar.gz
SUNJDK_URL=https://s3.amazonaws.com/ubuntu-ursus-packages/$SUNJDK_TGZ
[ ! -e /opt/tarfiles/$SUNJDK_TGZ -o ! -s /opt/tarfiles/$SUNJDK_TGZ ] && \
    sudo wget --no-check-certificate $SUNJDK_URL -O /opt/tarfiles/$SUNJDK_TGZ
sudo tar xzf /opt/tarfiles/$SUNJDK_TGZ -C /opt
sudo chown -R root:root /opt/jdk1.8.0_121
sudo umount -l /usr/lib/jvm/sunjdk || :
sudo mkdir -p /usr/lib/jvm/sunjdk
sudo mount --bind -o ro /opt/jdk1.8.0_121 /usr/lib/jvm/sunjdk

sudo mkdir /tmp/hd<commit_msg>fix typo<commit_after>#!/bin/bash

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

set -euxo pipefail

exec 1>&2

sudo apt-get update && sudo apt-get -y upgrade

sudo apt-get install -y csh wget curl vim git realpath tree htop libsnappy1v5 \
    lxc lvm2 xfsprogs pssh

pushd ~/var/yarn-ec2 > /dev/null

for vm in `sudo lxc-ls` ; do
    sudo lxc-stop -k -n $vm || :
    sudo lxc-destroy -f -n $vm
    sleep 0.1
done

sudo service lxc stop
sudo service lxc-net stop
sudo rm -f /var/lib/misc/dnsmasq.lxcbr0.leases
sudo killall -9 java || :
sleep 0.1

sudo rm -rf /tmp/Jetty*
sudo rm -rf /tmp/hadoop*
sudo rm -rf /tmp/hd*
sudo rm -rf /tmp/y*

sudo mkdir -p /opt/tarfiles
sudo chmod a+rx /opt/tarfiles
sudo rm -rf /opt/hadoop-*
sudo rm -rf /opt/jdk*

HADOOP_TGZ=hadoop-2.2.0.tar.gz
HADOOP_URL=https://s3.amazonaws.com/ubuntu-ursus-packages/$HADOOP_TGZ
[ ! -e /opt/tarfiles/$HADOOP_TGZ -o ! -s /opt/tarfiles/$HADOOP_TGZ ] && \
sudo wget --no-check-certificate $HADOOP_URL -O /opt/tarfiles/$HADOOP_TGZ
sudo tar xzf /opt/tarfiles/$HADOOP_TGZ -C /opt
sudo chown -R root:root /opt/hadoop-2.2.0
sudo umount -l /usr/local/hd || :
sudo mkdir -p /usr/local/hd
sudo mount --bind -o ro /opt/hadoop-2.2.0 /usr/local/hd
TAPACK_TGZ=advcc-sp1.tar.gz
TAPACK_URL=https://s3.amazonaws.com/ubuntu-ursus-packages/$TAPACK_TGZ
[ ! -e /opt/tarfiles/$TAPACK_TGZ -o ! -s /opt/tarfiles/$TAPACK_TGZ ] && \
    sudo wget --no-check-certificate $TAPACK_URL -O /opt/tarfiles/$TAPACK_TGZ
sudo tar xzf /opt/tarfiles/$TAPACK_TGZ -C /opt
sudo chown -R root:root /opt/advcc-sp1
sudo cp /opt/advcc-sp1/jobexe/* /opt/hadoop-2.2.0/
sudo mv /opt/hadoop-2.2.0/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-2.2.0.jar \
    /opt/hadoop-2.2.0/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-2.2.0.jar.origin
sudo cp /opt/advcc-sp1/hadoop-yarn-server-resourcemanager-2.2.0.jar /opt/hadoop-2.2.0/share/hadoop/yarn/
sudo cp /opt/advcc-sp1/hadoop-yarn-applications-mpirunner-2.2.0.jar /opt/hadoop-2.2.0/
sudo cp /opt/advcc-sp1/hadoop-yarn-applications-gpu-2.2.0.jar /opt/hadoop-2.2.0/

SUNJDK_TGZ=jdk-8u121-linux-x64.tar.gz
SUNJDK_URL=https://s3.amazonaws.com/ubuntu-ursus-packages/$SUNJDK_TGZ
[ ! -e /opt/tarfiles/$SUNJDK_TGZ -o ! -s /opt/tarfiles/$SUNJDK_TGZ ] && \
    sudo wget --no-check-certificate $SUNJDK_URL -O /opt/tarfiles/$SUNJDK_TGZ
sudo tar xzf /opt/tarfiles/$SUNJDK_TGZ -C /opt
sudo chown -R root:root /opt/jdk1.8.0_121
sudo umount -l /usr/lib/jvm/sunjdk || :
sudo mkdir -p /usr/lib/jvm/sunjdk
sudo mount --bind -o ro /opt/jdk1.8.0_121 /usr/lib/jvm/sunjdk

sudo mkdir /tmp/hd
{'fraction_non_alphanumeric': 0.13566009104704096, 'fraction_numerical': 0.025796661608497723, 'mean_word_length': 3.2966101694915255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1249838', 'n_tokens_mistral': 2842, 'n_tokens_neox': 2672, 'n_words': 678}