<commit_before>#!/bin/bash


# set deafults
HDFS_DIR="$(python -c 'import tabular_predDB.settings as S; print S.Hadoop.default_hdfs_dir')"
HDFS_URI="$(python -c 'import tabular_predDB.settings as S; print S.Hadoop.default_hdfs_uri')"
first_branch="master"
second_branch="engine_optimization"


# print script usage
usage() {
    cat <<EOF
usage: $0 options
    
    VMware automation functions
    
    OPTIONS:
    -h      Show this message
    -f      first_branch=$first_branch
    -s      second_branch=$second_branch
    -d      HDFS_DIR=$HDFS_DIR
    -u      HDFS_URI=$HDFS_URI
EOF
exit
}


#Process the arguments
while getopts hf:s:d:u: opt
do
    case "$opt" in
        h) usage;;
        f) first_branch=$OPTARG;;
        s) second_branch=$OPTARG;;
	d) HDFS_DIR=$OPTARG;;
        u) HDFS_URI=$OPTARG;;
    esac
done


make_binary_name() {
	which_branch=$1
	today_date=$(date +"%Y%M%d")
	which_engine_binary="hadoop_line_processor_${which_branch}_${today_date}.jar"
	echo $which_engine_binary
}


# make sure environment is correct
# export PYTHONPATH=~/tabular_predDB:$PYTHONPATH
repo_dir=$(python -c 'import tabular_predDB.settings as S; print S.path.this_repo_dir')
cd $repo_dir
sudo bash install_scripts/my_vpnc_connect.sh


# build both branches
for which_branch in $first_branch $second_branch; do
	git checkout $which_branch
	git pull
	cd $repo_dir
	make clean && make cython
	# FIXME: should break here if build fails
	cd tabular_predDB/binary_creation
	which_engine_binary=$(make_binary_name $which_branch)
	bash build_and_push.sh -b $which_engine_binary -d $HDFS_DIR -u $HDFS_URI


# runtime analysis
cd $repo_dir
git checkout master
cd tabular_predDB/timing_analysis
for which_branch in $first_branch $second_branch; do
	which_engine_binary=$(make_binary_name $which_branch)
	python automated_runtime_tests.py -do_remote --which_engine_binary $which_engine_binary \
	       --num_rows_list 1000 10000 --num_cols_list 128 256 --num_splits_list 8 16


# convergence analysis
cd $repo_dir
git checkout master
cd tabular_predDB/convergence_analysis
for which_branch in $first_branch $second_branch; do
	which_engine_binary=$(make_binary_name $which_branch)
	python automated_runtime_tests.py -do_remote --which_engine_binary $which_engine_binary \
	       --num_rows_list 1000 10000 --num_cols_list 128 256 --num_splits_list 8 16
	python automated_convergence_tests.py -do_remote --which_engine_binary $which_engine_binary \
		--num_rows_list 400 --num_cols_list 32 --num_clusters_list 5 --num_splits_list 4 \
		--max_mean_list 1 --n_steps 100 -do_plot
<commit_msg>BUGFIX: use a fixed build_date also, run both branches concurrently<commit_after>#!/bin/bash


# set deafults
HDFS_DIR="$(python -c 'import tabular_predDB.settings as S; print S.Hadoop.default_hdfs_dir')"
HDFS_URI="$(python -c 'import tabular_predDB.settings as S; print S.Hadoop.default_hdfs_uri')"
first_branch="master"
second_branch="engine_optimization"


# print script usage
usage() {
    cat <<EOF
usage: $0 options
    
    VMware automation functions
    
    OPTIONS:
    -h      Show this message
    -f      first_branch=$first_branch
    -s      second_branch=$second_branch
    -d      HDFS_DIR=$HDFS_DIR
    -u      HDFS_URI=$HDFS_URI
EOF
exit
}


#Process the arguments
while getopts hf:s:d:u: opt
do
    case "$opt" in
        h) usage;;
        f) first_branch=$OPTARG;;
        s) second_branch=$OPTARG;;
	d) HDFS_DIR=$OPTARG;;
        u) HDFS_URI=$OPTARG;;
    esac
done


make_binary_name() {
	which_branch=$1
	# build_date=$(date +"%Y%M%d")
	build_date=20130826
	which_engine_binary="${HDFS_DIR}/hadoop_line_processor_${which_branch}_${build_date}.jar"
	echo $which_engine_binary
}


# make sure environment is correct
# export PYTHONPATH=~/tabular_predDB:$PYTHONPATH
repo_dir=$(python -c 'import tabular_predDB.settings as S; print S.path.this_repo_dir')
cd $repo_dir
sudo bash install_scripts/my_vpnc_connect.sh


# build both branches
for which_branch in $first_branch $second_branch; do
	git checkout $which_branch
	git pull
	cd $repo_dir
	make clean && make cython
	# FIXME: should break here if build fails
	cd tabular_predDB/binary_creation
	which_engine_binary=$(make_binary_name $which_branch)
	bash build_and_push.sh -b $which_engine_binary -d $HDFS_DIR -u $HDFS_URI


# runtime analysis
cd $repo_dir
git checkout master
cd tabular_predDB/timing_analysis
for which_branch in $first_branch $second_branch; do
	which_engine_binary=$(make_binary_name $which_branch)
	python automated_runtime_tests.py -do_remote --which_engine_binary $which_engine_binary \
	       --num_rows_list 1000 --num_cols_list 16 32 --num_splits_list 2 4 &
	sleep 30
done
wait


# convergence analysis
cd $repo_dir
git checkout master
cd tabular_predDB/convergence_analysis
for which_branch in $first_branch $second_branch; do
	which_engine_binary=$(make_binary_name $which_branch)
	python automated_convergence_tests.py -do_remote --which_engine_binary $which_engine_binary \
		--num_rows_list 400 --num_cols_list 32 --num_clusters_list 5 --num_splits_list 4 \
		--max_mean_list 1 --n_steps 100 -do_plot &
	sleep 30
done
wait

{'fraction_non_alphanumeric': 0.09000195656427314, 'fraction_numerical': 0.01643513989434553, 'mean_word_length': 4.544468546637744, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5444185', 'n_tokens_mistral': 1908, 'n_tokens_neox': 1747, 'n_words': 437}