<commit_before>

CYCLE_REMOVING_TOOL="graph_pruning/graph_pruning.py"

CLEANING_TOOL="graph_pruning/cleaning.py"

EVAL_TOOL="eval/taxi_eval_archive/TExEval.jar"
EVAL_GOLD_STANDARD="eval/taxi_eval_archive/input/gold.taxo"
EVAL_ROOT=science
EVAL_JVM="-Xmx9000m"

OUTPUT_DIR=out

FILE_INPUT=$(basename "$1")
FILE_PRUNED_OUT=${FILE_INPUT}-pruned.csv
FILE_CLEANED_OUT=${FILE_PRUNED_OUT}-cleaned.csv

FILE_EVAL_TOOL_RESULT=${FILE_CLEANED_OUT}-evalresul.txt


echo Reading input: $1
echo Reading file: $FILE_INPUT
echo Output directory: $OUTPUT_DIR
echo

echo "======================================================================================================================"
echo "Cycle removing: python $CYCLE_REMOVING_TOOL $1 $OUTPUT_DIR/$FILE_PRUNED_OUT tarjan"
python $CYCLE_REMOVING_TOOL $1 $OUTPUT_DIR/$FILE_PRUNED_OUT tarjan
echo "Cycle removing finished. Written to: $OUTPUT_DIR/$FILE_PRUNED_OUT"
echo


echo "======================================================================================================================"
echo "Cleaning: python $CLEANING_TOOL $OUTPUT_DIR/$FILE_PRUNED_OUT $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_ROOT"
python $CLEANING_TOOL $OUTPUT_DIR/$FILE_PRUNED_OUT $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_ROOT
echo "Finished cleaning. Write output to: $OUTPUT_DIR/$FILE_CLEANED_OUT"
echo


echo "======================================================================================================================"
echo "Running eval-tool: java $EVAL_JVM -jar $EVAL_TOOL $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_GOLD_STANDARD $EVAL_ROOT $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT"
java $EVAL_JVM -jar $EVAL_TOOL $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_GOLD_STANDARD $EVAL_ROOT $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT
echo "Result of eval-tool written to: $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT"
echo

L_GOLD="$(wc -l $EVAL_GOLD_STANDARD | grep -o -E '^[0-9]+').0"
L_INPUT="$(wc -l $OUTPUT_DIR/$FILE_CLEANED_OUT | grep -o -E '^[0-9]+').0"
RECALL="$(tail -n 1 $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT | grep -o -E '[0-9]+[\.]?[0-9]*')"


PRECISION=$(echo("print $RECALL * $L_GOLD / $L_INPUT)" | python)
F1=$(echo "print(2 * $RECALL * $PRECISION / ($PRECISION + $RECALL))" | python)<commit_msg>fix matching paranthesis error in run.sh<commit_after>

CYCLE_REMOVING_TOOL="graph_pruning/graph_pruning.py"

CLEANING_TOOL="graph_pruning/cleaning.py"

EVAL_TOOL="eval/taxi_eval_archive/TExEval.jar"
EVAL_GOLD_STANDARD="eval/taxi_eval_archive/input/gold.taxo"
EVAL_ROOT=science
EVAL_JVM="-Xmx9000m"

OUTPUT_DIR=out

FILE_INPUT=$(basename "$1")
FILE_PRUNED_OUT=${FILE_INPUT}-pruned.csv
FILE_CLEANED_OUT=${FILE_PRUNED_OUT}-cleaned.csv

FILE_EVAL_TOOL_RESULT=${FILE_CLEANED_OUT}-evalresul.txt


echo Reading input: $1
echo Reading file: $FILE_INPUT
echo Output directory: $OUTPUT_DIR
echo

echo "======================================================================================================================"
echo "Cycle removing: python $CYCLE_REMOVING_TOOL $1 $OUTPUT_DIR/$FILE_PRUNED_OUT tarjan"
python $CYCLE_REMOVING_TOOL $1 $OUTPUT_DIR/$FILE_PRUNED_OUT tarjan
echo "Cycle removing finished. Written to: $OUTPUT_DIR/$FILE_PRUNED_OUT"
echo


echo "======================================================================================================================"
echo "Cleaning: python $CLEANING_TOOL $OUTPUT_DIR/$FILE_PRUNED_OUT $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_ROOT"
python $CLEANING_TOOL $OUTPUT_DIR/$FILE_PRUNED_OUT $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_ROOT
echo "Finished cleaning. Write output to: $OUTPUT_DIR/$FILE_CLEANED_OUT"
echo


echo "======================================================================================================================"
echo "Running eval-tool: java $EVAL_JVM -jar $EVAL_TOOL $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_GOLD_STANDARD $EVAL_ROOT $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT"
java $EVAL_JVM -jar $EVAL_TOOL $OUTPUT_DIR/$FILE_CLEANED_OUT $EVAL_GOLD_STANDARD $EVAL_ROOT $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT
echo "Result of eval-tool written to: $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT"
echo

L_GOLD="$(wc -l $EVAL_GOLD_STANDARD | grep -o -E '^[0-9]+').0"
L_INPUT="$(wc -l $OUTPUT_DIR/$FILE_CLEANED_OUT | grep -o -E '^[0-9]+').0"
RECALL="$(tail -n 1 $OUTPUT_DIR/$FILE_EVAL_TOOL_RESULT | grep -o -E '[0-9]+[\.]?[0-9]*')"


PRECISION=$(echo "print($RECALL * $L_GOLD / $L_INPUT)" | python)
F1=$(echo "print(2 * $RECALL * $PRECISION / ($PRECISION + $RECALL))" | python)
{'fraction_non_alphanumeric': 0.27413127413127414, 'fraction_numerical': 0.009538950715421303, 'mean_word_length': 8.941309255079007, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6279202', 'n_tokens_mistral': 1638, 'n_tokens_neox': 1518, 'n_words': 266}