<commit_before>#!/bin/bash

#Exit immediately if a command exits with a non-zero status
set -e

echo "Creating Hadoop users..."
sudo addgroup hadoop
sudo adduser --disabled-password --gecos "" --ingroup hadoop hduser

echo "Disabling IPv6 (not supported by Hadoop)..."
sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.default.disable_ipv6 = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.lo.disable_ipv6 = 1" >> /etc/sysctl.conf'

echo "Fetching Hadoop..."
HADOOP_DOWNLOAD_URL=http://apache.mirrors.spacedump.net/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz
HADOOP_TGZ=${HADOOP_DOWNLOAD_URL##*/}
HADOOP_PACKAGE_NAME=${HADOOP_TGZ%.tar.gz}
wget -q $HADOOP_DOWNLOAD_URL -O /tmp/$HADOOP_TGZ

echo "Installing $HADOOP_PACKAGE_NAME..."

sudo mkdir /opt/hadoop/
sudo tar xzf /tmp/$HADOOP_TGZ -C /opt/hadoop/
sudo ln -s /opt/hadoop/$HADOOP_PACKAGE_NAME /opt/hadoop/default
sudo chown hduser:hadoop -R /opt/hadoop/

# Create Hadoop temp directories for Namenode and Datanode
sudo mkdir -p /mnt/volume/hdfs/namenode
sudo mkdir -p /mnt/volume/hdfs/datanode
sudo chown hduser:hadoop -R /mnt/volume/hdfs

# Set environment
echo "export HADOOP_HOME=/opt/hadoop/" >> ~/.bashrc
echo "export PATH=$PATH:$HADOOP_HOME/bin" >> ~/.bashrc
echo "export PATH=$PATH:$HADOOP_HOME/sbin" >> ~/.bashrc
echo "export HADOOP_MAPRED_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export HADOOP_COMMON_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export HADOOP_HDFS_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export YARN_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native" >> ~/.bashrc
echo "export HADOOP_OPTS=\"-Djava.library.path=$HADOOP_HOME/lib\"" >> ~/.bashrc
sudo sh -c 'echo "JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /opt/hadoop/etc/hadoop/hadoop-env.sh'

echo "$HADOOP_PACKAGE_NAME installation complete."<commit_msg>fix hadoop env path<commit_after>#!/bin/bash

#Exit immediately if a command exits with a non-zero status
set -e

echo "Creating Hadoop users..."
sudo addgroup hadoop
sudo adduser --disabled-password --gecos "" --ingroup hadoop hduser

echo "Disabling IPv6 (not supported by Hadoop)..."
sudo sh -c 'echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.default.disable_ipv6 = 1" >> /etc/sysctl.conf'
sudo sh -c 'echo "net.ipv6.conf.lo.disable_ipv6 = 1" >> /etc/sysctl.conf'

echo "Fetching Hadoop..."
HADOOP_DOWNLOAD_URL=http://apache.mirrors.spacedump.net/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz
HADOOP_TGZ=${HADOOP_DOWNLOAD_URL##*/}
HADOOP_PACKAGE_NAME=${HADOOP_TGZ%.tar.gz}
wget -q $HADOOP_DOWNLOAD_URL -O /tmp/$HADOOP_TGZ

echo "Installing $HADOOP_PACKAGE_NAME..."

sudo mkdir /opt/hadoop/
sudo tar xzf /tmp/$HADOOP_TGZ -C /opt/hadoop/
sudo ln -s /opt/hadoop/$HADOOP_PACKAGE_NAME /opt/hadoop/default
sudo chown hduser:hadoop -R /opt/hadoop/

# Create Hadoop temp directories for Namenode and Datanode
sudo mkdir -p /mnt/volume/hdfs/namenode
sudo mkdir -p /mnt/volume/hdfs/datanode
sudo chown hduser:hadoop -R /mnt/volume/hdfs

# Set environment
echo "export HADOOP_HOME=/opt/hadoop/" >> ~/.bashrc
echo "export PATH=$PATH:$HADOOP_HOME/bin" >> ~/.bashrc
echo "export PATH=$PATH:$HADOOP_HOME/sbin" >> ~/.bashrc
echo "export HADOOP_MAPRED_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export HADOOP_COMMON_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export HADOOP_HDFS_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export YARN_HOME=$HADOOP_HOME" >> ~/.bashrc
echo "export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native" >> ~/.bashrc
echo "export HADOOP_OPTS=\"-Djava.library.path=$HADOOP_HOME/lib\"" >> ~/.bashrc
sudo sh -c 'echo "JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /opt/hadoop/default/etc/hadoop/hadoop-env.sh'

echo "$HADOOP_PACKAGE_NAME installation complete."
{'fraction_non_alphanumeric': 0.1543924250394529, 'fraction_numerical': 0.009994739610731194, 'mean_word_length': 4.617429837518464, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1539989', 'n_tokens_mistral': 1655, 'n_tokens_neox': 1554, 'n_words': 316}