<commit_before><?xml version="1.0" encoding="UTF-8" ?>
<!--
Copyright (c) 2017. Zuercher Hochschule fuer Angewandte Wissenschaften
All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.

Author: <NAME>
-->
<discocomponent>
    <properties>
        <!-- these properties can be set over HTTP parameters as well; they will be inserted automatically -->
        <property name="staticoutput" value="false" />
        <property name="included" value="false" />
		<property name="outputtype" value="append" />
    </properties>

	<dependencies>
		<dependency name="heat" state="start">
			<variable name="slavecount" />
			<variable name="mastername" />
			<variable name="slavename" />
		</dependency>
		<dependency name="shell" state="start" />
	</dependencies>

	<functions>
	</functions>

	<globaloutput />

	<output />

	<sparkstart><![CDATA[
# install Apache Spark on master node
deploymentLog "downloading Apache Spark to master node"
su ubuntu -c "sudo mkdir /home/ubuntu/spark"
cd /home/ubuntu/spark
chown ubuntu:ubuntu /home/ubuntu/spark
#su ubuntu -c "wget http://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz"
su ubuntu -c "wget http://reposerver/spark/spark-2.0.0-bin-hadoop2.7.tgz"

#su ubuntu -c "tar -xzf /home/ubuntu/spark/spark-2.0.0-bin-hadoop2.7.tgz"
su ubuntu -c "parallel-ssh -t 2000 -h /home/ubuntu/hosts.lst \"sudo sh -c \\\"echo \\\\\\\"SPARK_HOME=\\\\\\\\\\\\\\\"/usr/lib/spark/spark\\\\\\\\\\\\\\\"\\\\nJAVA_HOME=\\\\\\\\\\\\\\\"/usr/lib/java/jdk\\\\\\\\\\\\\\\"\\\\\\\" >> \\\/etc\\\/environment\\\"\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"sudo mkdir /usr/lib/spark\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"sudo chown ubuntu:ubuntu /usr/lib/spark/\""
su ubuntu -c "parallel-scp -h ~/hosts.lst ~/spark/spark-2.0.0-bin-hadoop2.7.tgz /usr/lib/spark"
su ubuntu -c "parallel-ssh -t 2000 -h /home/ubuntu/hosts.lst \"tar -xzf /usr/lib/spark/spark-2.0.0-bin-hadoop2.7.tgz\""
#su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"mv spark-2.0.0-bin-hadoop2.7 /usr/lib/spark/\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"mv spark-2.0.0-bin-hadoop2.7 /usr/lib/spark\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"ln -s /usr/lib/spark/spark-2.0.0-bin-hadoop2.7/ /usr/lib/spark/spark\""
cat - > /usr/lib/spark/spark/conf/slaves << 'EOF'
$mastername$
]]></sparkstart>


	<sparkend><![CDATA[EOF
su ubuntu -c "/usr/lib/spark/spark/sbin/start-master.sh"
su ubuntu -c "/usr/lib/spark/spark/sbin/start-slaves.sh"
deploymentLog "Spark installed and started"

]]></sparkend>


</discocomponent><commit_msg>added Spark History Server configuration and started Spark History Server at deployment<commit_after><?xml version="1.0" encoding="UTF-8" ?>
<!--
Copyright (c) 2017. Zuercher Hochschule fuer Angewandte Wissenschaften
All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.

Author: <NAME>
-->
<discocomponent>
    <properties>
        <!-- these properties can be set over HTTP parameters as well; they will be inserted automatically -->
        <property name="staticoutput" value="false" />
        <property name="included" value="false" />
		<property name="outputtype" value="append" />
    </properties>

	<dependencies>
		<dependency name="heat" state="start">
			<variable name="slavecount" />
			<variable name="mastername" />
			<variable name="slavename" />
		</dependency>
		<dependency name="shell" state="start" />
	</dependencies>

	<functions>
	</functions>

	<globaloutput />

	<output />

	<sparkstart><![CDATA[
# install Apache Spark on master node
deploymentLog "downloading Apache Spark to master node"
su ubuntu -c "sudo mkdir /home/ubuntu/spark"
cd /home/ubuntu/spark
chown ubuntu:ubuntu /home/ubuntu/spark
#su ubuntu -c "wget http://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz"
su ubuntu -c "wget http://reposerver/spark/spark-2.0.0-bin-hadoop2.7.tgz"

#su ubuntu -c "tar -xzf /home/ubuntu/spark/spark-2.0.0-bin-hadoop2.7.tgz"
su ubuntu -c "parallel-ssh -t 2000 -h /home/ubuntu/hosts.lst \"sudo sh -c \\\"echo \\\\\\\"SPARK_HOME=\\\\\\\\\\\\\\\"/usr/lib/spark/spark\\\\\\\\\\\\\\\"\\\\nJAVA_HOME=\\\\\\\\\\\\\\\"/usr/lib/java/jdk\\\\\\\\\\\\\\\"\\\\\\\" >> \\\/etc\\\/environment\\\"\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"sudo mkdir /usr/lib/spark\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"sudo chown ubuntu:ubuntu /usr/lib/spark/\""
su ubuntu -c "parallel-scp -h ~/hosts.lst ~/spark/spark-2.0.0-bin-hadoop2.7.tgz /usr/lib/spark"
su ubuntu -c "parallel-ssh -t 2000 -h /home/ubuntu/hosts.lst \"tar -xzf /usr/lib/spark/spark-2.0.0-bin-hadoop2.7.tgz\""
#su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"mv spark-2.0.0-bin-hadoop2.7 /usr/lib/spark/\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"mv spark-2.0.0-bin-hadoop2.7 /usr/lib/spark\""
su ubuntu -c "parallel-ssh -t 2000 -h ~/hosts.lst \"ln -s /usr/lib/spark/spark-2.0.0-bin-hadoop2.7/ /usr/lib/spark/spark\""
mkdir /var/log/sparkhistory
chown ubuntu:ubuntu /var/log/sparkhistory
cat - > /usr/lib/spark/spark/conf/spark-defaults.conf << 'EOF'
spark.history.fs.logDirectory	/var/log/sparkhistory
EOF

cat - > /usr/lib/spark/spark/conf/slaves << 'EOF'
$mastername$
]]></sparkstart>


	<sparkend><![CDATA[EOF
su ubuntu -c "/usr/lib/spark/spark/sbin/start-master.sh"
su ubuntu -c "/usr/lib/spark/spark/sbin/start-slaves.sh"

# start Spark History Server
su ubuntu -c "/usr/lib/spark/spark/sbin/start-history-server.sh"

deploymentLog "Spark installed and started"

]]></sparkend>


</discocomponent>
{'fraction_non_alphanumeric': 0.1795981452859351, 'fraction_numerical': 0.025965996908809894, 'mean_word_length': 3.2488509520682864, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 2, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '104831', 'n_tokens_mistral': 2539, 'n_tokens_neox': 2171, 'n_words': 613}