<commit_before>#!/bin/bash
# install hadoop (patched) and hhbase in /opt

if [ $EUID -ne 0 ]; then
    echo "you need to be root. please sudo."
    exit 1
fi


# TODO:
#  use mirrors for ${hadoop,hbase}_tar
#  also, apache seems to mirror only the very latest point release :-(

# set default variables.
prefix="/opt"
hadoop_version="hadoop-0.19.1"
hadoop_tar="$hadoop_version.tar.gz"
patch_1722="HADOOP-1722-branch-0.19.patch"
patch_5450="HADOOP-5450.patch"
hbase_version="hbase-0.19.2"
hbase_tar="$hbase_version.tar.gz"
#hadoop_release="http://mirrors.ukfast.co.uk/sites/ftp.apache.org/hadoop/core/$hadoop_version/$hadoop_tar"
hadoop_release="http://apache.mirror.infiniteconflict.com/hadoop//core/$hadoop_version/$hadoop_tar"
hadoop_1722="http://issues.apache.org/jira/secure/attachment/12401426/$patch_1722"
hadoop_5450="http://issues.apache.org/jira/secure/attachment/12401846/$patch_5450"
#hbase_release="http://mirrors.ukfast.co.uk/sites/ftp.apache.org/hadoop/hbase/$hbase_version/$hbase_tar"
hbase_release="http://apache.mirror.infiniteconflict.com/hadoop/hbase/$hbase_version/$hbase_tar"
install_log="$(pwd)/install_hadoop_hbase.log"
install_tmplog=$(mktemp /tmp/zohmg-log.XXXXXXXX)


# helpers.

# execute $1 and exit if it failed, displaying $2.
# TODO: the third argument is ignored. why? what was its purpose?
function exec_and_log() {
    # setup variables.
    command=$1

    [ "x" = "x$2" ] && msg="Error: Could not execute $command." || msg=$2

    # execute and log to intermediate.
    # FIXME: doesn't seem to log stderr
    # FIXME: doesn't retain return code when using pipes
    $1 # | tee "$install_tmplog" | cat
    ret=$?

    # log.
    # FIXME: does append not work?  =>  cat $install_tmplog >> $install_log
    cat "$install_log" "$install_tmplog" >"$install_log.new"
    mv "$install_log.new" "$install_log"

    # check exit code.
    if [ $ret -ne 0 ]; then
        echo
        echo $msg
        exit 1
    fi
}


function usage() {
    echo "Usage: $(basename $0) [options]"
    echo "Options:"
    echo "    --download-only      Only download software, do not build."
    echo "    --files=FILES        Directory with already downloaded files."
    echo "                         Files are downloaded to /tmp/zohmg-deps.XXXXXX."
    echo "    --hadoop-only        Installs only Apache Hadoop."
    echo "                         Cannot be used with --hbase-only."
    echo "    --hbase-only         Installs only Apache HBase."
    echo "                         Cannot be used with --hadoop-only."
    echo "    --help               Prints this help and exits."
    echo "    --keep-files         Keeps downloaded files."
    echo "    --no-config          Do not configure Hadoop or HBase."
    echo "    --prefix=PREFIX      Changes installation prefix to PREFIX."
    echo "                         Defaults to /opt."
}


# parse arguments.
while [ $1 ]; do
    opt=$(echo $1 | sed 's/=.*//')
    arg=$(echo $1 | sed 's/.*=//')
    case $opt in
        "--download-only")
            download_only="true"
            ;;
        "--files")
            files="$arg"
            keep_files="true" # do not delete files we did not download.
            ;;
        "--hadoop-only")
            if [ $hbase_only ]; then
                echo "Error: --hbase-only and --hadoop-only used at the same time."
                usage
                exit 1
            fi
            hadoop_only="true"
            ;;
        "--hbase-only")
            if [ $hadoop_only ]; then
                echo "Error: --hbase-only and --hadoop-only used at the same time."
                usage
                exit 1
            fi
            hbase_only="true"
            ;;
        "--help")
            usage
            exit 0
            ;;
        "--keep-files")
            keep_files="true"
            ;;
        "--no-config")
            no_config="true"
            ;;
        "--prefix")
            prefix="$arg"
            ;;
        *)
            if [ ! "x" = "x$opt" ]; then
                echo "Unknown argument: $opt"
                usage
                exit 1
            fi
            ;;
    esac
    shift
done



if [ -f $prefix ]; then<commit_msg>install_hadoop_hbase.sh> make sure the user knows what's up.<commit_after>#!/bin/bash
# install hadoop (patched) and hhbase in /opt

if [ $EUID -ne 0 ]; then
    echo "you need to be root. please sudo."
    exit 1
fi



# TODO:
#  use mirrors for ${hadoop,hbase}_tar
#  also, apache seems to mirror only the very latest point release :-(

# set default variables.
prefix="/opt"
hadoop_version="hadoop-0.19.1"
hadoop_tar="$hadoop_version.tar.gz"
patch_1722="HADOOP-1722-branch-0.19.patch"
patch_5450="HADOOP-5450.patch"
hbase_version="hbase-0.19.2"
hbase_tar="$hbase_version.tar.gz"
#hadoop_release="http://mirrors.ukfast.co.uk/sites/ftp.apache.org/hadoop/core/$hadoop_version/$hadoop_tar"
hadoop_release="http://apache.mirror.infiniteconflict.com/hadoop//core/$hadoop_version/$hadoop_tar"
hadoop_1722="http://issues.apache.org/jira/secure/attachment/12401426/$patch_1722"
hadoop_5450="http://issues.apache.org/jira/secure/attachment/12401846/$patch_5450"
#hbase_release="http://mirrors.ukfast.co.uk/sites/ftp.apache.org/hadoop/hbase/$hbase_version/$hbase_tar"
hbase_release="http://apache.mirror.infiniteconflict.com/hadoop/hbase/$hbase_version/$hbase_tar"
install_log="$(pwd)/install_hadoop_hbase.log"
install_tmplog=$(mktemp /tmp/zohmg-log.XXXXXXXX)


# helpers.

# execute $1 and exit if it failed, displaying $2.
# TODO: the third argument is ignored. why? what was its purpose?
function exec_and_log() {
    # setup variables.
    command=$1

    [ "x" = "x$2" ] && msg="Error: Could not execute $command." || msg=$2

    # execute and log to intermediate.
    # FIXME: doesn't seem to log stderr
    # FIXME: doesn't retain return code when using pipes
    $1 # | tee "$install_tmplog" | cat
    ret=$?

    # log.
    # FIXME: does append not work?  =>  cat $install_tmplog >> $install_log
    cat "$install_log" "$install_tmplog" >"$install_log.new"
    mv "$install_log.new" "$install_log"

    # check exit code.
    if [ $ret -ne 0 ]; then
        echo
        echo $msg
        exit 1
    fi
}


function usage() {
    echo "Usage: $(basename $0) [options]"
    echo "Options:"
    echo "    --download-only      Only download software, do not build."
    echo "    --files=FILES        Directory with already downloaded files."
    echo "                         Files are downloaded to /tmp/zohmg-deps.XXXXXX."
    echo "    --hadoop-only        Installs only Apache Hadoop."
    echo "                         Cannot be used with --hbase-only."
    echo "    --hbase-only         Installs only Apache HBase."
    echo "                         Cannot be used with --hadoop-only."
    echo "    --help               Prints this help and exits."
    echo "    --keep-files         Keeps downloaded files."
    echo "    --no-config          Do not configure Hadoop or HBase."
    echo "    --prefix=PREFIX      Changes installation prefix to PREFIX."
    echo "                         Defaults to /opt."
}


# parse arguments.
while [ $1 ]; do
    opt=$(echo $1 | sed 's/=.*//')
    arg=$(echo $1 | sed 's/.*=//')
    case $opt in
        "--download-only")
            download_only="true"
            ;;
        "--files")
            files="$arg"
            keep_files="true" # do not delete files we did not download.
            ;;
        "--hadoop-only")
            if [ $hbase_only ]; then
                echo "Error: --hbase-only and --hadoop-only used at the same time."
                usage
                exit 1
            fi
            hadoop_only="true"
            ;;
        "--hbase-only")
            if [ $hadoop_only ]; then
                echo "Error: --hbase-only and --hadoop-only used at the same time."
                usage
                exit 1
            fi
            hbase_only="true"
            ;;
        "--help")
            usage
            exit 0
            ;;
        "--keep-files")
            keep_files="true"
            ;;
        "--no-config")
            no_config="true"
            ;;
        "--prefix")
            prefix="$arg"
            ;;
        *)
            if [ ! "x" = "x$opt" ]; then
                echo "Unknown argument: $opt"
                usage
                exit 1
            fi
            ;;
    esac
    shift
done


# make sure the user knows what's up.
echo "this script will download, patch and install hadoop & hbase in $prefix"
echo "any key to continue, CTRL-C to abort."
read

if [ -f $prefix ]; then
{'fraction_non_alphanumeric': 0.12906598513011153, 'fraction_numerical': 0.017890334572490705, 'mean_word_length': 1.8582337317397077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6488848', 'n_tokens_mistral': 2801, 'n_tokens_neox': 2605, 'n_words': 694}