<commit_before>#! /bin/sh

cd /home/vagrant/

echo "Add Cloudera repositories"
wget -c http://archive.cloudera.com/cdh5/one-click-install/precise/amd64/cdh5-repository_1.0_all.deb
sudo dpkg -i cdh5-repository_1.0_all.deb
curl -s http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | sudo apt-key add -

sudo apt-get update

echo "Install Java"
sudo apt-get install --force-yes --yes openjdk-7-jdk

echo "Install Hadoop with YARN"
sudo apt-get install --yes hadoop-conf-pseudo

echo "Starting Hadoop and Verifying it is Working Properly"
dpkg -L hadoop-conf-pseudo

echo "Format the NameNode."
sudo -u hdfs hdfs namenode -format

echo "Start HDFS"
for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do sudo service $x start ; done

# echo "Remove the old /tmp if it exists:"
# sudo -u hdfs hadoop fs -rm -r /tmp

echo "Create the new directories and set permissions:"
sudo -u hdfs hadoop fs -mkdir -p /tmp/hadoop-yarn/staging/history/done_intermediate
sudo -u hdfs hadoop fs -chown -R mapred:mapred /tmp/hadoop-yarn/staging
sudo -u hdfs hadoop fs -chmod -R 1777 /tmp
sudo -u hdfs hadoop fs -mkdir -p /var/log/hadoop-yarn
sudo -u hdfs hadoop fs -chown yarn:mapred /var/log/hadoop-yarn

echo "Verify the HDFS File Structure:"
sudo -u hdfs hadoop fs -ls -R /

echo "Start YARN"
sudo service hadoop-yarn-resourcemanager start
sudo service hadoop-yarn-nodemanager start
sudo service hadoop-mapreduce-historyserver start

echo "Create User Directories"
sudo -u hdfs hadoop fs -mkdir /user
sudo -u hdfs hadoop fs -chown vagrant /user
sudo -u hdfs hadoop fs -mkdir /user/vagrant
sudo -u hdfs hadoop fs -chown vagrant /user/vagrant

echo "Install Hive"
sudo apt-get install --yes hive
sudo apt-get install --yes hive-hcatalog

echo "Install HBase"
sudo apt-get install --yes hbase-master

echo "Install Pig"
sudo apt-get install --yes pig

echo "Install Mahout"
sudo apt-get install --yes mahout

sudo su
echo "export HIVE_OPTS=\"-hiveconf mapred.job.tracker=yarn -hiveconf fs.default.name=hdfs://localhost:8020 -hiveconf hive.metastore.warehouse.dir=hdfs:///user/vagrant/warehouse -hiveconf javax.jdo.option.ConnectionURL=jdbc:derby:;databaseName=/tmp/metastore_db;create=true\"\nexport JAVA_HOME=\"/usr/lib/jvm/java-1.7.0-openjdk-amd64/\"\nexport CLASSPATH=\"\`hadoop classpath\`:/usr/lib/pig/*\"" > /etc/profile.d/envvars.sh
echo "LC_ALL=\"en_US.UTF-8\"" > /etc/default/locale<commit_msg>Update install.sh<commit_after>#! /bin/sh

cd /home/vagrant/

echo "Add Cloudera repositories"
wget -c http://archive.cloudera.com/cdh5/one-click-install/precise/amd64/cdh5-repository_1.0_all.deb
sudo dpkg -i cdh5-repository_1.0_all.deb
curl -s http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | sudo apt-key add -

sudo apt-get update

echo "Install Java"
sudo apt-get install --force-yes --yes openjdk-7-jdk

echo "Install Hadoop with YARN"
sudo apt-get install --yes hadoop-conf-pseudo

echo "Starting Hadoop and Verifying it is Working Properly"
dpkg -L hadoop-conf-pseudo

echo "Format the NameNode."
sudo -u hdfs hdfs namenode -format

echo "Start HDFS"
for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do sudo service $x start ; done

# echo "Remove the old /tmp if it exists:"
# sudo -u hdfs hadoop fs -rm -r /tmp

echo "Create the new directories and set permissions:"
sudo -u hdfs hadoop fs -mkdir -p /tmp/hadoop-yarn/staging/history/done_intermediate
sudo -u hdfs hadoop fs -chown -R mapred:mapred /tmp/hadoop-yarn/staging
sudo -u hdfs hadoop fs -chmod -R 1777 /tmp
sudo -u hdfs hadoop fs -mkdir -p /var/log/hadoop-yarn
sudo -u hdfs hadoop fs -chown yarn:mapred /var/log/hadoop-yarn

echo "Verify the HDFS File Structure:"
sudo -u hdfs hadoop fs -ls -R /

echo "Start YARN"
sudo service hadoop-yarn-resourcemanager start
sudo service hadoop-yarn-nodemanager start
sudo service hadoop-mapreduce-historyserver start

echo "Create User Directories"
sudo -u hdfs hadoop fs -mkdir /user
sudo -u hdfs hadoop fs -chown vagrant /user
sudo -u hdfs hadoop fs -mkdir /user/vagrant
sudo -u hdfs hadoop fs -chown vagrant /user/vagrant

echo "Install Hive"
sudo apt-get install --yes hive
sudo apt-get install --yes hive-hcatalog

echo "Install HBase"
sudo apt-get install --yes hbase-master

echo "Install Pig"
sudo apt-get install --yes pig

echo "Install Mahout"
sudo apt-get install --yes mahout

sudo su
echo "export HIVE_OPTS=\"-hiveconf mapreduce.jobtracker.address=localhost:8021 -hiveconf fs.defaultFS=hdfs://localhost:8020 -hiveconf hive.metastore.warehouse.dir=hdfs:///user/vagrant/warehouse -hiveconf javax.jdo.option.ConnectionURL=jdbc:derby:;databaseName=/tmp/metastore_db;create=true\"\nexport JAVA_HOME=\"/usr/lib/jvm/java-1.7.0-openjdk-amd64/\"\nexport CLASSPATH=\"\`hadoop classpath\`:/usr/lib/pig/*\"" > /etc/profile.d/envvars.sh
echo "LC_ALL=\"en_US.UTF-8\"" > /etc/default/locale
{'fraction_non_alphanumeric': 0.11993348576179588, 'fraction_numerical': 0.012055705674495947, 'mean_word_length': 3.8704453441295548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2063299', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1799, 'n_words': 530}