<commit_before>#!/bin/bash

#Paralelism
PARA=""
#Number of repeats
ROUNDS=""
#Input Path (will be converted to hdfs path)
IN=""
#Output Path (will be converted to hdfs path)
OUT=""
#CSV output path (directory must be existing)
CSV=""
#Vertex grouping keys
VGK=""
#Edge grouping keys
EGK=""
#Use vertex labels
UVL=""
#Use edge labels
UEL=""
#Vertex aggregate functions
VAGG=""
#Vertex aggregate keys
VAK=""
#Vertex aggregate result keys
VARK=""
#Edge aggregate functions
EAGG=""
#Edge aggregate keys
EAK=""
#Edge aggregate result keys
EARK=""
#Used jar file
JAR_FILE=""
#Used benchmarking class
CLASS=""
#HDFS root directory
HDFS=""
#FLINK root directory
FLINK=""

while read LINE
do 
[[ "$LINE" =~ ^#.*$ ]] && continue
LINE="$(echo -e "${LINE}" | tr -d '[[:space:]]')"
IFS=':' read -ra LINE <<< "$LINE"
KEY=${LINE[0]}

case ${KEY} in
    flink_root)	 FLINK="${LINE[1]}";;
    hdfs_root)  HDFS="${LINE[1]}";;
    class)   CLASS="${LINE[1]}";;
    jar)     JAR_FILE="${LINE[1]}";;
    parallelism) PARA="${LINE[1]}";;
    rounds)      ROUNDS="${LINE[1]}";;
    input)	 IN="${LINE[1]}";;
    output)	 OUT="${LINE[1]}";;
    csv)     	 CSV="${LINE[1]}";;
    vgk)	 VGK="-vgk ${LINE[1]}";;
    egk)	 EGK="-egk ${LINE[1]}";;
    uvl) 	 UVL="-uvl";;
    uel)	 UEL="-uel";;
    vagg)     	 VAGG="${LINE[1]}";;
    vak)	 VAK="-vak ${LINE[1]}";;
    vark)	 VARK="-vark ${LINE[1]}";;
    eagg)	 EAGG="${LINE[1]}";;
    eak)	 EAK="-eak ${LINE[1]]}";;
    eark)	 EARK="-eark ${LINE[1]}";
esac

done < grouping.conf

IFS=',' 
read -ra PARA <<< "$PARA"
read -ra IN <<< "$IN"

unset IFS

for P in "${PARA[@]}"
do
	for I in "${IN[@]}"
	do
		for ((R=1; R<=$ROUNDS; R++))
		do
		    echo "Benchmark"
		    echo "========="
		    echo "INPUT: ${I}"
		    echo "OUTPUT: ${OUT}"
		    echo "PARALLELISM: ${P}"
 		    echo "========="
		    ${HDFS}/bin/hdfs dfs -rm -r ${OUT}
		    INPUT="hdfs://${I}"
		    OUTPUT="hdfs://${OUT}"
		    AGGS="-vagg ${VAGG} ${VAK} ${VARK} -eagg ${EAGG} ${EAK} ${EARK}"
		    ARGS="-csv ${CSV} ${VGK} ${EGK} ${AGGS}"
            ${FLINK}/bin/flink run -p ${P} -c ${CLASS} ${JAR_FILE} -i ${INPUT} -o ${OUTPUT} ${ARGS}
		done
	done
done<commit_msg>consider uvl and uel options in grouping.sh, fixes #243<commit_after>#!/bin/bash

#Paralelism
PARA=""
#Number of repeats
ROUNDS=""
#Input Path (will be converted to hdfs path)
IN=""
#Output Path (will be converted to hdfs path)
OUT=""
#CSV output path (directory must be existing)
CSV=""
#Vertex grouping keys
VGK=""
#Edge grouping keys
EGK=""
#Use vertex labels
UVL=""
#Use edge labels
UEL=""
#Vertex aggregate functions
VAGG=""
#Vertex aggregate keys
VAK=""
#Vertex aggregate result keys
VARK=""
#Edge aggregate functions
EAGG=""
#Edge aggregate keys
EAK=""
#Edge aggregate result keys
EARK=""
#Used jar file
JAR_FILE=""
#Used benchmarking class
CLASS=""
#HDFS root directory
HDFS=""
#FLINK root directory
FLINK=""

while read LINE
do 
[[ "$LINE" =~ ^#.*$ ]] && continue
LINE="$(echo -e "${LINE}" | tr -d '[[:space:]]')"
IFS=':' read -ra LINE <<< "$LINE"
KEY=${LINE[0]}

case ${KEY} in
    flink_root)	 FLINK="${LINE[1]}";;
    hdfs_root)  HDFS="${LINE[1]}";;
    class)   CLASS="${LINE[1]}";;
    jar)     JAR_FILE="${LINE[1]}";;
    parallelism) PARA="${LINE[1]}";;
    rounds)      ROUNDS="${LINE[1]}";;
    input)	 IN="${LINE[1]}";;
    output)	 OUT="${LINE[1]}";;
    csv)     	 CSV="${LINE[1]}";;
    vgk)	 VGK="-vgk ${LINE[1]}";;
    egk)	 EGK="-egk ${LINE[1]}";;
    uvl) 	 UVL="-uvl";;
    uel)	 UEL="-uel";;
    vagg)     	 VAGG="${LINE[1]}";;
    vak)	 VAK="-vak ${LINE[1]}";;
    vark)	 VARK="-vark ${LINE[1]}";;
    eagg)	 EAGG="${LINE[1]}";;
    eak)	 EAK="-eak ${LINE[1]]}";;
    eark)	 EARK="-eark ${LINE[1]}";
esac

done < grouping.conf

IFS=',' 
read -ra PARA <<< "$PARA"
read -ra IN <<< "$IN"

unset IFS

for P in "${PARA[@]}"
do
	for I in "${IN[@]}"
	do
		for ((R=1; R<=$ROUNDS; R++))
		do
		    echo "Benchmark"
		    echo "========="
		    echo "INPUT: ${I}"
		    echo "OUTPUT: ${OUT}"
		    echo "PARALLELISM: ${P}"
 		    echo "========="
		    ${HDFS}/bin/hdfs dfs -rm -r ${OUT}
		    INPUT="hdfs://${I}"
		    OUTPUT="hdfs://${OUT}"
		    AGGS="-vagg ${VAGG} ${VAK} ${VARK} -eagg ${EAGG} ${EAK} ${EARK}"
		    ARGS="-csv ${CSV} ${UVL} ${UEL} ${VGK} ${EGK} ${AGGS}"
            ${FLINK}/bin/flink run -p ${P} -c ${CLASS} ${JAR_FILE} -i ${INPUT} -o ${OUTPUT} ${ARGS}
		done
	done
done
{'fraction_non_alphanumeric': 0.24851122308749427, 'fraction_numerical': 0.009390746678882272, 'mean_word_length': 3.4200404858299596, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6708178', 'n_tokens_mistral': 1892, 'n_tokens_neox': 1778, 'n_words': 475}