<commit_before>#! /bin/bash


## BASIC FUNCTIONS ##
function package_exists() {
    return dpkg -l "$1" &> /dev/null
}

function usage() {
    echo ""
    echo "Usage: sudo -E bash install-ignite-app.sh";
    echo "This script does NOT require Ambari username and password";
    exit 132;
}

## IMPORT HELPER MODULE ##
wget -O /tmp/HDInsightUtilities-v01.sh -q https://hdiconfigactions.blob.core.windows.net/linuxconfigactionmodulev01/HDInsightUtilities-v01.sh && source /tmp/HDInsightUtilities-v01.sh && rm -f /tmp/HDInsightUtilities-v01.sh

## GET AMBARI USERNAME AND PASSWORD ##
USERID=$(echo -e "import hdinsight_common.Constants as Constants\nprint Constants.AMBARI_WATCHDOG_USERNAME" | python)
echo "USERID=$USERID"
PASSWD=$(echo -e "import hdinsight_common.ClusterManifestParser as ClusterManifestParser\nimport hdinsight_common.Constants as Constants\nimport base64\nbase64pwd = ClusterManifestParser.parse_local_manifest().ambari_users.usersmap[Constants.AMBARI_WATCHDOG_USERNAME].password\nprint base64.b64decode(base64pwd)" | python)

PRIMARYHEADNODE=`get_primary_headnode`
SECONDARYHEADNODE=`get_secondary_headnode`

## Check if values retrieved are empty, if yes, exit with error ##
if [[ -z $PRIMARYHEADNODE ]]; then
	echo "Could not determine primary headnode."
	exit 139
fi
    
if [[ -z $SECONDARYHEADNODE ]]; then
	echo "Could not determine secondary headnode."
	exit 140
fi

## DEFINE ENVIRONMENT VARIABLES ##
export HADOOP_HOME="/usr/hdp/current/hadoop-client";
export HADOOP_COMMON_HOME="/usr/hdp/current/hadoop-client";
export HADOOP_HDFS_HOME="/usr/hdp/current/hadoop-hdfs-client";
export HADOOP_MAPRED_HOME="/usr/hdp/current/hadoop-mapreduce-client";

IGNITE_BINARY="apache-ignite-hadoop-1.7.0-bin";
IGNITE_BINARY_URI="https://www.apache.org/dist/ignite/1.7.0/$IGNITE_BINARY.zip";
IGNITE_TMPFOLDER=/tmp/ignite
export IGNITE_HOME_DIR="/hadoop/ignite";
export IGNITE_HOME="$IGNITE_HOME_DIR/$IGNITE_BINARY";


AMBARICONFIGS_SH=/var/lib/ambari-server/resources/scripts/configs.sh
PORT=8080
ACTIVEAMBARIHOST=headnodehost

export AMBARI_ADMIN=$USERID
export AMBARI_PWD=$<PASSWORD>

echo "Beginning main.."
## ASSISTING FUNCTIONS ##
function checkHostNameAndSetClusterName() {
    fullHostName=$(hostname -f)
    echo "fullHostName=$fullHostName"
    CLUSTERNAME=$(sed -n -e 's/.*\.\(.*\)-ssh.*/\1/p' <<< $fullHostName)
    if [ -z "$CLUSTERNAME" ]; then
        CLUSTERNAME=$(echo -e "import hdinsight_common.ClusterManifestParser as ClusterManifestParser\nprint ClusterManifestParser.parse_local_manifest().deployment.cluster_name" | python)
        if [ $? -ne 0 ]; then
            echo "[ERROR] Cannot determine cluster name. Exiting!"
            exit 133
        fi
    fi
    echo "Cluster Name=$CLUSTERNAME"
    
    export AMBARI_HOST=$fullHostName
    export AMBARI_CLUSTER=$CLUSTERNAME
    
    export FS_DEFAULT_DFS=`$AMBARICONFIGS_SH -u $USERID -p $PASSWD -port $PORT get $ACTIVEAMBARIHOST $CLUSTERNAME core-site | grep -o '"wasb:.*"' | sed 's/"//g'`
    export WORKER_NODES=(`curl -k -s -u $USERID:$PASSWD "http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/hosts" | grep -o '"wn.*"' | sed 's/"//g'`)
}

function validateUsernameAndPassword() {
    coreSiteContent=$(bash $AMBARICONFIGS_SH -u $USERID -p $PASSWD -port $PORT get $ACTIVEAMBARIHOST $CLUSTERNAME core-site)
    if [[ $coreSiteContent == *"[ERROR]"* && $coreSiteContent == *"Bad credentials"* ]]; then
        echo "[ERROR] Username and password are invalid. Exiting!"
        exit 134
    fi
}

function updateAmbariConfigs() {
    echo "AMBARI HOST = $AMBARI_HOST"
    echo "AMBARI CLUSTER = $AMBARI_CLUSTER"
    echo "ACTIVE AMBARI HOST = $ACTIVEAMBARIHOST"
    echo "CLUSTERNAME = $CLUSTERNAME"
    
    updateResult=$(bash $AMBARICONFIGS_SH -u $USERID -p $PASSWD -p $PORT set $ACTIVEAMBARIHOST $CLUSTERNAME core-site "fs.igfs.impl" "org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem")
    
    if [[ $updateResult != *"Tag:version"* ]] && [[ $updateResult == *"[ERROR]"* ]]; then
        echo "[ERROR] Failed to update core-site for property: 'fs.igfs.impl', Exiting!"
        echo $updateResult
        exit 135
    fi
    
    echo "Updated core-site.xml with fs.igfs.impl = org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem"
    
    updateResult=$(bash $AMBARICONFIGS_SH -u $USERID -p $PASSWD set $ACTIVEAMBARIHOST $CLUSTERNAME core-site "fs.AbstractFileSystem.igfs.impl" "org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem")
    
    if [[ $updateResult != *"Tag:version"* ]] && [[ $updateResult == *"[ERROR]"* ]]; then
        echo "[ERROR] Failed to update core-site for property: 'fs.AbstractFileSystem.igfs.impl', Exiting!"
        echo $updateResult
        exit 135
    fi
    
    echo "Updated core-site.xml with fs.AbstractFileSystem.igfs.impl = org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem"
}

function stopServiceViaRest() {
    if [ -z "$1" ]; then
        echo "Need service name to stop service"
        exit 136
    fi
    SERVICENAME=$1
    echo "Stopping $SERVICENAME"
    curl -u $USERID:$PASSWD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Stop Service for Hue installation"}, "Body": {"ServiceInfo": {"state": "INSTALLED"}}}' http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/services/$SERVICENAME
}

function startServiceViaRest() {
    if [ -z "$1" ]; then
        echo "Need service name to start service"
        exit 136
    fi
    sleep 2
    SERVICENAME=$1
    echo "Starting $SERVICENAME"
    startResult=$(curl -u $USERID:$PASSWD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Start Service for Hue installation"}, "Body": {"ServiceInfo": {"state": "STARTED"}}}' http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/services/$SERVICENAME)
    if [[ $startResult == *"500 Server Error"* || $startResult == *"internal system exception occurred"* ]]; then
        sleep 60
        echo "Retry starting $SERVICENAME"
        startResult=$(curl -u $USERID:$PASSWD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Start Service for Hue installation"}, "Body": {"ServiceInfo": {"state": "STARTED"}}}' http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/services/$SERVICENAME)
    fi
    echo $startResult
}

function downloadAndUnzipApacheIgnite() {
    #KILL APACHE IGNITE IF RUNNING
	ignitepid=`ps -ef | grep ignite | grep default-config.xml | awk '{print $2}'`
	if [ ! -z "$ignitepid" ]; then
		echo "killing running Apache Ignite instance"
		kill -9 $ignitepid
	fi
	
	#DELETE APACHE IGNITE BEFORE INSTALLING
	if [ -d "$IGNITE_HOME" ]; then 
		echo "Removing existing Ignite binaries: $IGNITE_HOME_DIR/$IGNITE_BINARY"
		rm -r $IGNITE_HOME_DIR/$IGNITE_BINARY
		rm $IGNITE_HOME_DIR/$IGNITE_BINARY.zip; 
	fi
	
	echo "Downloading Apache Ignite"
	mkdir -p $IGNITE_HOME_DIR
	wget -P $IGNITE_HOME_DIR $IGNITE_BINARY_URI;
	
	echo "Unzipping Apache Ignite"
	unzip $IGNITE_HOME_DIR/$IGNITE_BINARY.zip -d $IGNITE_HOME_DIR;
	
	echo "Remove Apache Ignite zip file"
	rm $IGNITE_HOME_DIR/$IGNITE_BINARY.zip;
}

function updateApacheSparkConfig(){
	echo "backing up spark-env.sh to $IGNITE_HOME"
	cp $SPARK_HOME/conf/spark-env.sh $IGNITE_HOME/config/spark-env.sh.backup;
		
su spark <<'EOG'
sed -i -e '$a\' $SPARK_HOME/conf/spark-env.sh
		
IGNITE_BINARY="apache-ignite-hadoop-1.7.0-bin";
export IGNITE_HOME_DIR="/hadoop/ignite";
export IGNITE_HOME="$IGNITE_HOME_DIR/$IGNITE_BINARY";
export HADOOP_HOME="/usr/hdp/current/hadoop-client";
export HADOOP_COMMON_HOME="/usr/hdp/current/hadoop-client";
export HADOOP_HDFS_HOME="/usr/hdp/current/hadoop-hdfs-client";
export HADOOP_MAPRED_HOME="/usr/hdp/current/hadoop-mapreduce-client";
		
sed -i -e '$a\' $SPARK_HOME/conf/spark-env.sh
#append ignite libs to spark-env.sh
cat <<EOT >> $SPARK_HOME/conf/spark-env.sh
		
IGNITE_BINARY="apache-ignite-hadoop-1.7.0-bin"
IGNITE_HOME="/hadoop/ignite/$IGNITE_BINARY"
IGNITE_LIBS="\${IGNITE_HOME}/libs/*"
for file in \${IGNITE_LIBS}
do
if [ -d \${file} ] && [ "\${file}" != "\${IGNITE_HOME}"/libs/optional ]; then
IGNITE_LIBS=\${IGNITE_LIBS}:\${file}/*
fi
done
export SPARK_CLASSPATH=\$SPARK_CLASSPATH:\$IGNITE_LIBS		
EOT
EOG	<commit_msg>add indentation<commit_after>#! /bin/bash


## BASIC FUNCTIONS ##
function package_exists() {
    return dpkg -l "$1" &> /dev/null
}

function usage() {
    echo ""
    echo "Usage: sudo -E bash install-ignite-app.sh";
    echo "This script does NOT require Ambari username and password";
    exit 132;
}

## IMPORT HELPER MODULE ##
wget -O /tmp/HDInsightUtilities-v01.sh -q https://hdiconfigactions.blob.core.windows.net/linuxconfigactionmodulev01/HDInsightUtilities-v01.sh && source /tmp/HDInsightUtilities-v01.sh && rm -f /tmp/HDInsightUtilities-v01.sh

## GET AMBARI USERNAME AND PASSWORD ##
USERID=$(echo -e "import hdinsight_common.Constants as Constants\nprint Constants.AMBARI_WATCHDOG_USERNAME" | python)
echo "USERID=$USERID"
PASSWD=$(echo -e "import hdinsight_common.ClusterManifestParser as ClusterManifestParser\nimport hdinsight_common.Constants as Constants\nimport base64\nbase64pwd = ClusterManifestParser.parse_local_manifest().ambari_users.usersmap[Constants.AMBARI_WATCHDOG_USERNAME].password\nprint base64.b64decode(base64pwd)" | python)

PRIMARYHEADNODE=`get_primary_headnode`
SECONDARYHEADNODE=`get_secondary_headnode`

## Check if values retrieved are empty, if yes, exit with error ##
if [[ -z $PRIMARYHEADNODE ]]; then
	echo "Could not determine primary headnode."
	exit 139
fi
    
if [[ -z $SECONDARYHEADNODE ]]; then
	echo "Could not determine secondary headnode."
	exit 140
fi

## DEFINE ENVIRONMENT VARIABLES ##
export HADOOP_HOME="/usr/hdp/current/hadoop-client";
export HADOOP_COMMON_HOME="/usr/hdp/current/hadoop-client";
export HADOOP_HDFS_HOME="/usr/hdp/current/hadoop-hdfs-client";
export HADOOP_MAPRED_HOME="/usr/hdp/current/hadoop-mapreduce-client";

IGNITE_BINARY="apache-ignite-hadoop-1.7.0-bin";
IGNITE_BINARY_URI="https://www.apache.org/dist/ignite/1.7.0/$IGNITE_BINARY.zip";
IGNITE_TMPFOLDER=/tmp/ignite
export IGNITE_HOME_DIR="/hadoop/ignite";
export IGNITE_HOME="$IGNITE_HOME_DIR/$IGNITE_BINARY";


AMBARICONFIGS_SH=/var/lib/ambari-server/resources/scripts/configs.sh
PORT=8080
ACTIVEAMBARIHOST=headnodehost

export AMBARI_ADMIN=$USERID
export AMBARI_PWD=$<PASSWORD>

echo "Beginning main.."
## ASSISTING FUNCTIONS ##
function checkHostNameAndSetClusterName() {
    fullHostName=$(hostname -f)
    echo "fullHostName=$fullHostName"
    CLUSTERNAME=$(sed -n -e 's/.*\.\(.*\)-ssh.*/\1/p' <<< $fullHostName)
    if [ -z "$CLUSTERNAME" ]; then
        CLUSTERNAME=$(echo -e "import hdinsight_common.ClusterManifestParser as ClusterManifestParser\nprint ClusterManifestParser.parse_local_manifest().deployment.cluster_name" | python)
        if [ $? -ne 0 ]; then
            echo "[ERROR] Cannot determine cluster name. Exiting!"
            exit 133
        fi
    fi
    echo "Cluster Name=$CLUSTERNAME"
    
    export AMBARI_HOST=$fullHostName
    export AMBARI_CLUSTER=$CLUSTERNAME
    
    export FS_DEFAULT_DFS=`$AMBARICONFIGS_SH -u $USERID -p $PASSWD -port $PORT get $ACTIVEAMBARIHOST $CLUSTERNAME core-site | grep -o '"wasb:.*"' | sed 's/"//g'`
    export WORKER_NODES=(`curl -k -s -u $USERID:$PASSWD "http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/hosts" | grep -o '"wn.*"' | sed 's/"//g'`)
}

function validateUsernameAndPassword() {
    coreSiteContent=$(bash $AMBARICONFIGS_SH -u $USERID -p $PASSWD -port $PORT get $ACTIVEAMBARIHOST $CLUSTERNAME core-site)
    if [[ $coreSiteContent == *"[ERROR]"* && $coreSiteContent == *"Bad credentials"* ]]; then
        echo "[ERROR] Username and password are invalid. Exiting!"
        exit 134
    fi
}

function updateAmbariConfigs() {
    echo "AMBARI HOST = $AMBARI_HOST"
    echo "AMBARI CLUSTER = $AMBARI_CLUSTER"
    echo "ACTIVE AMBARI HOST = $ACTIVEAMBARIHOST"
    echo "CLUSTERNAME = $CLUSTERNAME"
    
    updateResult=$(bash $AMBARICONFIGS_SH -u $USERID -p $PASSWD -p $PORT set $ACTIVEAMBARIHOST $CLUSTERNAME core-site "fs.igfs.impl" "org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem")
    
    if [[ $updateResult != *"Tag:version"* ]] && [[ $updateResult == *"[ERROR]"* ]]; then
        echo "[ERROR] Failed to update core-site for property: 'fs.igfs.impl', Exiting!"
        echo $updateResult
        exit 135
    fi
    
    echo "Updated core-site.xml with fs.igfs.impl = org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem"
    
    updateResult=$(bash $AMBARICONFIGS_SH -u $USERID -p $PASSWD set $ACTIVEAMBARIHOST $CLUSTERNAME core-site "fs.AbstractFileSystem.igfs.impl" "org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem")
    
    if [[ $updateResult != *"Tag:version"* ]] && [[ $updateResult == *"[ERROR]"* ]]; then
        echo "[ERROR] Failed to update core-site for property: 'fs.AbstractFileSystem.igfs.impl', Exiting!"
        echo $updateResult
        exit 135
    fi
    
    echo "Updated core-site.xml with fs.AbstractFileSystem.igfs.impl = org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem"
}

function stopServiceViaRest() {
    if [ -z "$1" ]; then
        echo "Need service name to stop service"
        exit 136
    fi
    SERVICENAME=$1
    echo "Stopping $SERVICENAME"
    curl -u $USERID:$PASSWD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Stop Service for Hue installation"}, "Body": {"ServiceInfo": {"state": "INSTALLED"}}}' http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/services/$SERVICENAME
}

function startServiceViaRest() {
    if [ -z "$1" ]; then
        echo "Need service name to start service"
        exit 136
    fi
    sleep 2
    SERVICENAME=$1
    echo "Starting $SERVICENAME"
    startResult=$(curl -u $USERID:$PASSWD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Start Service for Hue installation"}, "Body": {"ServiceInfo": {"state": "STARTED"}}}' http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/services/$SERVICENAME)
    if [[ $startResult == *"500 Server Error"* || $startResult == *"internal system exception occurred"* ]]; then
        sleep 60
        echo "Retry starting $SERVICENAME"
        startResult=$(curl -u $USERID:$PASSWD -i -H 'X-Requested-By: ambari' -X PUT -d '{"RequestInfo": {"context" :"Start Service for Hue installation"}, "Body": {"ServiceInfo": {"state": "STARTED"}}}' http://$ACTIVEAMBARIHOST:$PORT/api/v1/clusters/$CLUSTERNAME/services/$SERVICENAME)
    fi
    echo $startResult
}

function downloadAndUnzipApacheIgnite() {
    #KILL APACHE IGNITE IF RUNNING
	ignitepid=`ps -ef | grep ignite | grep default-config.xml | awk '{print $2}'`
	if [ ! -z "$ignitepid" ]; then
		echo "killing running Apache Ignite instance"
		kill -9 $ignitepid
	fi
	
	#DELETE APACHE IGNITE BEFORE INSTALLING
	if [ -d "$IGNITE_HOME" ]; then 
		echo "Removing existing Ignite binaries: $IGNITE_HOME_DIR/$IGNITE_BINARY"
		rm -r $IGNITE_HOME_DIR/$IGNITE_BINARY
		rm $IGNITE_HOME_DIR/$IGNITE_BINARY.zip; 
	fi
	
	echo "Downloading Apache Ignite"
	mkdir -p $IGNITE_HOME_DIR
	wget -P $IGNITE_HOME_DIR $IGNITE_BINARY_URI;
	
	echo "Unzipping Apache Ignite"
	unzip $IGNITE_HOME_DIR/$IGNITE_BINARY.zip -d $IGNITE_HOME_DIR;
	
	echo "Remove Apache Ignite zip file"
	rm $IGNITE_HOME_DIR/$IGNITE_BINARY.zip;
}

function updateApacheSparkConfig(){
	echo "backing up spark-env.sh to $IGNITE_HOME"
	cp $SPARK_HOME/conf/spark-env.sh $IGNITE_HOME/config/spark-env.sh.backup;
		
su spark <<'EOG'
	sed -i -e '$a\' $SPARK_HOME/conf/spark-env.sh

	IGNITE_BINARY="apache-ignite-hadoop-1.7.0-bin";
	export IGNITE_HOME_DIR="/hadoop/ignite";
	export IGNITE_HOME="$IGNITE_HOME_DIR/$IGNITE_BINARY";
	export HADOOP_HOME="/usr/hdp/current/hadoop-client";
	export HADOOP_COMMON_HOME="/usr/hdp/current/hadoop-client";
	export HADOOP_HDFS_HOME="/usr/hdp/current/hadoop-hdfs-client";
	export HADOOP_MAPRED_HOME="/usr/hdp/current/hadoop-mapreduce-client";

	sed -i -e '$a\' $SPARK_HOME/conf/spark-env.sh
	#append ignite libs to spark-env.sh
	cat <<EOT >> $SPARK_HOME/conf/spark-env.sh

	IGNITE_BINARY="apache-ignite-hadoop-1.7.0-bin"
	IGNITE_HOME="/hadoop/ignite/$IGNITE_BINARY"
	IGNITE_LIBS="\${IGNITE_HOME}/libs/*"
	for file in \${IGNITE_LIBS}
	do
	if [ -d \${file} ] && [ "\${file}" != "\${IGNITE_HOME}"/libs/optional ]; then
	IGNITE_LIBS=\${IGNITE_LIBS}:\${file}/*
	fi
	done
	export SPARK_CLASSPATH=\$SPARK_CLASSPATH:\$IGNITE_LIBS		
	EOT
EOG	
{'fraction_non_alphanumeric': 0.1416330028155221, 'fraction_numerical': 0.010527604357938548, 'mean_word_length': 4.250321336760925, 'pattern_counts': {'":': 24, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2461287', 'n_tokens_mistral': 6207, 'n_tokens_neox': 5875, 'n_words': 1239}