<commit_before>#!/bin/bash

if [ $1 == "tera" ]; then
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar teragen 256 /service/data/teragen-output'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar terasort /service/data/teragen-output /service/data/terasort-output'
elif [ $1 == "text" ]; then
	su drydock -c 'wget http://www.gutenberg.org/cache/epub/20417/pg20417.txt -P /tmp/example-text/'
	su drydock -c 'wget http://www.gutenberg.org/cache/epub/5000/pg5000.txt -P /tmp/example-text/'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf dfs -copyFromLocal /tmp/example-text /service/data/example-text'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordcount /service/data/example-text /service/data/wordcount-output'/
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordmean /service/data/example-text /service/data/wordmean-output'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordmedian /service/data/example-text /service/data/wordmedian-output'
elif [ $1 == "hive" ]; then
	su drydock -c 'wget http://files.grouplens.org/datasets/movielens/ml-100k/u.data -P /tmp/movielens/'
	su drydock -c '/service/packages/hadoop/bin/hdfs dfs -mkdir -p /service/data/movielens'
	su drydock -c '/service/packages/hadoop/bin/hdfs dfs -copyFromLocal /tmp/movielens/u.data /service/data/movielens'
	su drydock -c '/service/packages/hive/bin/hive -f /service/scripts/createtable.sql'
elif [ $1 == "dfs" ]; then
	su drydock -c '/service/packages/hadoop/bin/hadoop dfsadmin -report'
fi<commit_msg>add gluster hadoop example<commit_after>#!/bin/bash

if [ $1 == "tera" ]; then
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar teragen 256 /service/data/teragen-output'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar terasort /service/data/teragen-output /service/data/terasort-output'
elif [ $1 == "text" ]; then
	su drydock -c 'wget http://www.gutenberg.org/cache/epub/20417/pg20417.txt -P /tmp/example-text/'
	su drydock -c 'wget http://www.gutenberg.org/cache/epub/5000/pg5000.txt -P /tmp/example-text/'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf dfs -copyFromLocal /tmp/example-text /service/data/example-text'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordcount /service/data/example-text /service/data/wordcount-output'/
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordmean /service/data/example-text /service/data/wordmean-output'
	su drydock -c '/service/packages/hadoop/bin/hadoop --config /service/conf jar /service/packages/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.3.0.jar wordmedian /service/data/example-text /service/data/wordmedian-output'
elif [ $1 == "hive" ]; then
	su drydock -c 'wget http://files.grouplens.org/datasets/movielens/ml-100k/u.data -P /tmp/movielens/'
	su drydock -c '/service/packages/hadoop/bin/hdfs dfs -mkdir -p /service/data/movielens'
	su drydock -c '/service/packages/hadoop/bin/hdfs dfs -copyFromLocal /tmp/movielens/u.data /service/data/movielens'
	su drydock -c '/service/packages/hive/bin/hive -f /service/scripts/createtable.sql'
elif [ $1 == "gluster" ]; then
	su drydock -c 'wget http://files.grouplens.org/datasets/movielens/ml-100k/u.data -P /service/data/movielens/'
	su drydock -c '/service/packages/hive/bin/hive -f /service/scripts/createtable.sql'
elif [ $1 == "dfs" ]; then
	su drydock -c '/service/packages/hadoop/bin/hadoop dfsadmin -report'
fi
{'fraction_non_alphanumeric': 0.15720127214902316, 'fraction_numerical': 0.02044525215810995, 'mean_word_length': 3.963923337091319, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7312110', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1533, 'n_words': 283}