<commit_before>#!/bin/bash
#
# To run this script the ec2-ami-tools have to be installed on your system. 
# Afterwards the following command will create a new instance and execute this script right after system startup. 
# It should be good for TLC development afterwards.
#
# ec2-run-instances -m --key <EMAIL> --instance-type m2.4xlarge --user-data-file /path/to/ProvisionEC2.sh ami-c162a9a8
#

# switch to mount to use the instance ephemeral storage rather than ESB
cd /mnt

# download non pkg mgmt provided tools in background
# no package mgmt for Eclipse profiler, hence install manually into /opt
(wget -q http://www.yourkit.com/download/yjp-10.0.4-linux.tar.bz2  && cd /opt && tar xfj /mnt/yjp-10.0.4-linux.tar.bz2) &

# no package mgmt for yourkit profiler, hence install manually into /opt
(wget -q ftp://mirror.cc.vt.edu/pub/eclipse/technology/epp/downloads/release/indigo/SR1/eclipse-rcp-indigo-SR1-linux-gtk-x86_64.tar.gz && cd /opt && tar xfz /mnt/eclipse-rcp-indigo-SR1-linux-gtk-x86_64.tar.gz) & 

# apache maven 3
(wget -q http://apache.osuosl.org//maven/binaries/apache-maven-3.0.3-bin.tar.gz && cd /opt && tar xfz /mnt/apache-maven-3.0.3-bin.tar.gz) &

# aapche ant
(wget -q http://newverhost.com/pub//ant/binaries/apache-ant-1.8.2-bin.tar.gz && cd /opt && tar xfz /mnt/apache-ant-1.8.2-bin.tar.gz) &

# create user kuppe and setup public key
useradd --home /mnt/kuppe -m kuppe -s /bin/bash -G admin,sudo
ln -s /mnt/kuppe /home/kuppe
mkdir /home/kuppe/.ssh
# personal one to ssh into the box
echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAy4sKgMk2QS8wKze690bmar4cJXz2/AGJr36uJ8gds6zAhM9cyQ4GEuOgOORAOWcUAg3HZI65GdCqtHE+P4t9P5Qi1bE+d24ZL0Ebg6z5P8f8XvSId7Gd/p7YXgCcOJj4nxRHRr3fmHvLw+s0PLCaCg5uqShOBhyxbD8H8pgh4WqxUhHtAeIV+OhuazgLjCYL9b2DFSUtad6j+c+podcZGOJq+DLhP3NWJI5eiO2u7PuX06sRCTo4UxZlP+bvjhU7tH/VRIg9R7CO9TvmMykJEAwa6G5vf4kCFeh9f0pYA26UUFg8z0P2ki7+Z0zHXD3w7zbDYzZ9snFY9RtajtNkXw== <EMAIL>" >> /home/kuppe/.ssh/authorized_keys
# ec2 keypair to ssh out
echo "-----BEGIN RSA PRIVATE KEY-----
<KEY>
" > /home/kuppe/.ssh/id_rsa
echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDY+EiWONhPeuupD26mtePrFDzel5ewHcLow0FfdCNnTyRdXRJVo8iyDROP5YyNfphmgezsRcv62sXAft4sqKoVw14luX9XPjNWDARsTbPOyhBTsJe6qMm/8nZ78zGbL8EEWVE52l3rx2MWYy67VwBaAP5szVVpmj+7VJ4fCh/1Vp9ORgOxwgzdpkt2qx8Tn5qVkjOwxqpGsANsaXLmfdRo9eXFXO845Ok7bytPqdwJsSoB2tRUH8rwHrVPQsJsXYnTQFKf8mhf21uxhdsnHg4jWEk1My/XI+x/mjR8ksQWcpjumVAFw7flMD1ZYBijo3CR+b2GtV4uKTxjf8bbEcBn <EMAIL>@<EMAIL>" > /home/kuppe/.ssh/id_rsa.pub
chmod 600 /home/kuppe/.ssh/id_rsa

# fix permission because steps are executed by root
chown -R kuppe:kuppe /home/kuppe
echo "kuppe ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

# create user lamport
useradd -m lamport -s /bin/bash -G admin,sudo
mkdir /home/lamport/.ssh
echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAodOzPQlOIq+8ZDb75L0FR6VeMV/B+COXvkXPiwI4kEXkiadiEkynv7GIwVx3AfryvRtG2gYBUDccNuATVmz7HfhxKGlGOCLY4aZw3qIsMTTfe3nlQ0cRbU4q4npDwiPEQW8MUZe9zfBpPUL1eakZkTSpZaAOTLgYqRgOobLpSKk= lamport@SVC-LAPORT-1" > /home/lamport/.ssh/authorized_keys
chown -R lamport:lamport /home/lamport
echo "lamport ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

# add x2go ubuntu ppa repository to package manager apt
add-apt-repository ppa:x2go/stable -y

# add cloudera hadoop repository to package manager
wget http://archive.cloudera.com/one-click-install/maverick/cdh3-repository_1.0_all.deb
dpkg -i cdh3-repository_1.0_all.deb

# update package index and install basic packages needed
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get upgrade -y
apt-get --no-install-recommends install openjdk-7-jdk unzip mc htop sysstat apache2 munin munin-node munin-java-plugins munin-plugins-extra gnome-core gdm gnome-session-fallback firefox git git-svn tightvncserver xorg x2goserver x2goserver-xsession sshfs rsync -y

# remove overlay scrollbar. it messes with eclipse
apt-get purge overlay-scrollbar liboverlay-scrollbar-0.2-0 liboverlay-scrollbar3-0.2-0 -y

# activate extra munin stats
rm /etc/munin/plugins/apache_*
rm /etc/munin/plugins/munin_*
rm /etc/munin/plugins/http_*
rm /etc/munin/plugins/fw_*
# for jmx plugin to work, the vm has to be started with -D properties to listen on port 5400
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_ClassesLoaded
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_ClassesLoadedTotal
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_ClassesUnloaded
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_CompilationTimeTotal
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_GCCount
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_GCTime
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_CurrentThreadCpuTime
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_CurrentThreadUserTime<commit_msg>split X deployment from base packages, add juju pkg<commit_after>#!/bin/bash
#
# To run this script the ec2-ami-tools have to be installed on your system. 
# Afterwards the following command will create a new instance and execute this script right after system startup. 
# It should be good for TLC development afterwards.
#
# ec2-run-instances -m --key <EMAIL> --instance-type m2.4xlarge --user-data-file /path/to/ProvisionEC2.sh ami-c162a9a8
#

# switch to mount to use the instance ephemeral storage rather than ESB
cd /mnt

# download non pkg mgmt provided tools in background
# no package mgmt for Eclipse profiler, hence install manually into /opt
(wget -q http://www.yourkit.com/download/yjp-10.0.4-linux.tar.bz2  && cd /opt && tar xfj /mnt/yjp-10.0.4-linux.tar.bz2) &

# no package mgmt for yourkit profiler, hence install manually into /opt
(wget -q ftp://mirror.cc.vt.edu/pub/eclipse/technology/epp/downloads/release/indigo/SR1/eclipse-rcp-indigo-SR1-linux-gtk-x86_64.tar.gz && cd /opt && tar xfz /mnt/eclipse-rcp-indigo-SR1-linux-gtk-x86_64.tar.gz) & 

# apache maven 3
(wget -q http://apache.osuosl.org//maven/binaries/apache-maven-3.0.3-bin.tar.gz && cd /opt && tar xfz /mnt/apache-maven-3.0.3-bin.tar.gz) &

# aapche ant
(wget -q http://newverhost.com/pub//ant/binaries/apache-ant-1.8.2-bin.tar.gz && cd /opt && tar xfz /mnt/apache-ant-1.8.2-bin.tar.gz) &

# create user kuppe and setup public key
useradd --home /mnt/kuppe -m kuppe -s /bin/bash -G admin,sudo
ln -s /mnt/kuppe /home/kuppe
mkdir /home/kuppe/.ssh
# personal one to ssh into the box
echo "ssh-rsa A<KEY>2EAAAABIwAAAQEAy4sKgMk2QS8wKze690bmar4cJXz2/AGJr36uJ8gds6zAhM9cyQ4GEuOgOORAOWcUAg3HZI65GdCqtHE+P4t9P5Qi1bE+d24ZL0Ebg6z5P8f8XvSId7Gd/p7YXgCcOJj4nxRHRr3fmHvLw+s0PLCaCg5uqShOBhyxbD8H8pgh4WqxUhHtAeIV+OhuazgLjCYL9b2DFSUtad6j+c+podcZGOJq+DLhP3NWJI5eiO2u7PuX06sRCTo4UxZlP+bvjhU7tH/VRIg9R7CO9TvmMykJEAwa6G5vf4kCFeh9f0pYA26UUFg8z0P2ki7+Z0zHXD3w7zbDYzZ9snFY9RtajtNkXw== <EMAIL>" >> /home/kuppe/.ssh/authorized_keys
# ec2 keypair to ssh out
echo "-----BEGIN RSA PRIVATE KEY-----
<KEY>
" > /home/kuppe/.ssh/id_rsa
echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDY+EiWONhPeuupD26mtePrFDzel5ewHcLow0FfdCNnTyRdXRJVo8iyDROP5YyNfphmgezsRcv62sXAft4sqKoVw14luX9XPjNWDARsTbPOyhBTsJe6qMm/8nZ78zGbL8EEWVE52l3rx2MWYy67VwBaAP5szVVpmj+7VJ4fCh/1Vp9ORgOxwgzdpkt2qx8Tn5qVkjOwxqpGsANsaXLmfdRo9eXFXO845Ok7bytPqdwJsSoB2tRUH8rwHrVPQsJsXYnTQFKf8mhf21uxhdsnHg4jWEk1My/XI+x/mjR8ksQWcpjumVAFw7flMD1ZYBijo3CR+b2GtV4uKTxjf8bbEcBn k<EMAIL>@<EMAIL>" > /home/kuppe/.ssh/id_rsa.pub
chmod 600 /home/kuppe/.ssh/id_rsa

# fix permission because steps are executed by root
chown -R kuppe:kuppe /home/kuppe
echo "kuppe ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

# create user lamport
useradd -m lamport -s /bin/bash -G admin,sudo
mkdir /home/lamport/.ssh
echo "ssh-rsa <KEY>vRtG2gYBUDccNuATVmz7HfhxKGlGOCLY4aZw3qIsMTTfe3nlQ0cRbU4q4npDwiPEQW8MUZe9zfBpPUL1eakZkTSpZaAOTLgYqRgOobLpSKk= lamport@SVC-LAPORT-1" > /home/lamport/.ssh/authorized_keys
chown -R lamport:lamport /home/lamport
echo "lamport ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

# add x2go ubuntu ppa repository to package manager apt
add-apt-repository ppa:x2go/stable -y

# add cloudera hadoop repository to package manager
wget http://archive.cloudera.com/one-click-install/maverick/cdh3-repository_1.0_all.deb
dpkg -i cdh3-repository_1.0_all.deb

# update package index and install basic packages needed
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get upgrade -y
apt-get --no-install-recommends install openjdk-7-jdk juju unzip mc htop sysstat apache2 munin munin-node munin-java-plugins munin-plugins-extra git git-svn sshfs rsync -y

# if UI/X needed
apt-get --no-install-recommends install gnome-core gdm gnome-session-fallback firefox tightvncserver xorg x2goserver x2goserver-xsession -y

# remove overlay scrollbar. it messes with eclipse
apt-get purge overlay-scrollbar liboverlay-scrollbar-0.2-0 liboverlay-scrollbar3-0.2-0 -y

# activate extra munin stats
rm /etc/munin/plugins/apache_*
rm /etc/munin/plugins/munin_*
rm /etc/munin/plugins/http_*
rm /etc/munin/plugins/fw_*
# for jmx plugin to work, the vm has to be started with -D properties to listen on port 5400
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_ClassesLoaded
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_ClassesLoadedTotal
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_ClassesUnloaded
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_CompilationTimeTotal
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_GCCount
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_GCTime
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_CurrentThreadCpuTime
ln -s /usr/share/munin/plugins/jmx_ /etc/munin/plugins/jmx_CurrentThreadUserTime
{'fraction_non_alphanumeric': 0.10704933486959255, 'fraction_numerical': 0.04409762228972452, 'mean_word_length': 4.875692307692308, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7301893', 'n_tokens_mistral': 4563, 'n_tokens_neox': 4262, 'n_words': 753}