<commit_before>#1/bin/bash

# uncomment the following line for debug mode
set -x
set -e
set -o nounset

EXPAND=true
QUOTEFILTER=true
KEEPBEST=true

: ${KBPREPO:?"Need to set KBPREPO to path to working copy of kbp"}
: ${KBPOPENREPO:?"Need to set KBPOPENREPO to path to working copy of kbp-2014-event-arguments"}
: ${PARTICIPANTS:?"Need to set PARTICIPANTS to /nfs/mercury-04/u22/kbp-2015/eval_analysis/interim_2015/systemsOutput"}
: ${ASSESSMENTS:?"Need to set $ASSESSMENTS to /nfs/mercury-04/u22/kbp-2015/eval_analysis/interim_2015/assessments"}

EVALDIR=${KBPOPENREPO}/output/interim_2015
LOG=$EVALDIR/log/interim_2015 

echo "Using working copy $KBPOPENREPO"
echo "Writing log to $EVALDIR/log"

# clear previous run, if any
echo "Output will be written to $EVALDIR"
echo "Clearing previous output, if any"
rm -rf $EVALDIR

echo "Creating output directory"
mkdir -p $EVALDIR/log

# uncompress participant submissions
PARTICIPANTCOPY=$EVALDIR/participantSubmissions
mkdir -p $PARTICIPANTCOPY
echo "Copying participant submissions from $PARTICIPANTS to $PARTICIPANTCOPY"
#shopt -s extglob
cp -r $PARTICIPANTS/* $PARTICIPANTCOPY
#shopt -u extglob


#shopt -u nullglob

# copy LDC assessments
LDCCOPY=$EVALDIR/ldcAssessment
ASSESSDIR=$LDCCOPY/data/LDC_assessments
#mkdir -p $LDCCOPY
#echo "extracting LDC assessments from $ASSESSMENTS to $LDCCOPY"
#tar xzf $ASSESSMENTS -C $LDCCOPY --strip-components=2
echo "Copying LDC assessments from $ASSESSMENTS to $ASSESSDIR"
mkdir -p $ASSESSDIR
cp -r $ASSESSMENTS/{annotation,linkingStore} $ASSESSDIR/

# apply realis expansion to LDC assessments
if [ "$EXPAND" = true ]; then
    echo "Expanded assessment store using realis assessments..."
    $KBPOPENREPO/kbp-events2014-bin/target/appassembler/bin/expandByRealis $KBPOPENREPO/params/interim_2015/expand.params > $LOG.expand.log

    mkdir -p $EVALDIR/expanded/linkingStore
    cp $ASSESSDIR/linkingStore/* $EVALDIR/expanded/linkingStore
fi

# quote filter participant submissions
if [ "$QUOTEFILTER" = true ]; then
    echo "Applying quote filter to submissions..."
    $KBPOPENREPO/kbp-events2014-bin/target/appassembler/bin/applyQuoteFilter $KBPOPENREPO/params/interim_2015/quoteFilter.params > $LOG.quoteFilter.log
fi

# do keepBest
if [ "$KEEPBEST" = true ] ; then
    echo "Applying keep best to assessment stores"

    for f in $EVALDIR/quoteFiltered/*; do
      if [ -d ${f} ]; then
        sysId=$(basename $f)
        mkdir -p $EVALDIR/keepBest/$sysId

        echo "inputStore: $f" > $KBPOPENREPO/params/interim_2015/generated/keepBest_${sysId}.params
	echo "outputStore: $EVALDIR/keepBest/$sysId" >> $KBPOPENREPO/params/interim_2015/generated/keepBest_${sysId}.params
	echo "keepInferenceCases: false" >> $KBPOPENREPO/params/interim_2015/generated/keepBest_${sysId}.params
	echo "outputLayout: KBP_EA_2015" >> $KBPOPENREPO/params/interim_2015/generated/keepBest_${sysId}.params

        $KBPOPENREPO/kbp-events2014-bin/target/appassembler/bin/keepOnlyBestResponses $KBPOPENREPO/params/interim_2015/generated/keepBest_${sysId}.params > $LOG.keepBest_${sysId}.log
      fi
    done
fi

<commit_msg>revise to conventional way in bash, to write a small file<commit_after>#1/bin/bash

# uncomment the following line for debug mode
set -x
set -e
set -o nounset

EXPAND=true
QUOTEFILTER=true
KEEPBEST=true

: ${KBPREPO:?"Need to set KBPREPO to path to working copy of kbp"}
: ${KBPOPENREPO:?"Need to set KBPOPENREPO to path to working copy of kbp-2014-event-arguments"}
: ${PARTICIPANTS:?"Need to set PARTICIPANTS to /nfs/mercury-04/u22/kbp-2015/eval_analysis/interim_2015/systemsOutput"}
: ${ASSESSMENTS:?"Need to set $ASSESSMENTS to /nfs/mercury-04/u22/kbp-2015/eval_analysis/interim_2015/assessments"}

EVALDIR=${KBPOPENREPO}/output/interim_2015
LOG=$EVALDIR/log/interim_2015 

echo "Using working copy $KBPOPENREPO"
echo "Writing log to $EVALDIR/log"

# clear previous run, if any
echo "Output will be written to $EVALDIR"
echo "Clearing previous output, if any"
rm -rf $EVALDIR

echo "Creating output directory"
mkdir -p $EVALDIR/log

# uncompress participant submissions
PARTICIPANTCOPY=$EVALDIR/participantSubmissions
mkdir -p $PARTICIPANTCOPY
echo "Copying participant submissions from $PARTICIPANTS to $PARTICIPANTCOPY"
#shopt -s extglob
cp -r $PARTICIPANTS/* $PARTICIPANTCOPY
#shopt -u extglob


#shopt -u nullglob

# copy LDC assessments
LDCCOPY=$EVALDIR/ldcAssessment
ASSESSDIR=$LDCCOPY/data/LDC_assessments
#mkdir -p $LDCCOPY
#echo "extracting LDC assessments from $ASSESSMENTS to $LDCCOPY"
#tar xzf $ASSESSMENTS -C $LDCCOPY --strip-components=2
echo "Copying LDC assessments from $ASSESSMENTS to $ASSESSDIR"
mkdir -p $ASSESSDIR
cp -r $ASSESSMENTS/{annotation,linkingStore} $ASSESSDIR/

# apply realis expansion to LDC assessments
if [ "$EXPAND" = true ]; then
    echo "Expanded assessment store using realis assessments..."
    $KBPOPENREPO/kbp-events2014-bin/target/appassembler/bin/expandByRealis $KBPOPENREPO/params/interim_2015/expand.params > $LOG.expand.log

    mkdir -p $EVALDIR/expanded/linkingStore
    cp $ASSESSDIR/linkingStore/* $EVALDIR/expanded/linkingStore
fi

# quote filter participant submissions
if [ "$QUOTEFILTER" = true ]; then
    echo "Applying quote filter to submissions..."
    $KBPOPENREPO/kbp-events2014-bin/target/appassembler/bin/applyQuoteFilter $KBPOPENREPO/params/interim_2015/quoteFilter.params > $LOG.quoteFilter.log
fi

# do keepBest
if [ "$KEEPBEST" = true ] ; then
    echo "Applying keep best to assessment stores"

    for f in $EVALDIR/quoteFiltered/*; do
      if [ -d ${f} ]; then
        sysId=$(basename $f)
        mkdir -p $EVALDIR/keepBest/$sysId

	cat <<EOF > $KBPOPENREPO/params/interim_2015/generated/keepBest_${sysId}.params
	inputStore: $f
	outputStore: $EVALDIR/keepBest/$sysId
	keepInferenceCases: false
	outputLayout: KBP_EA_2015
	EOF

        $KBPOPENREPO/kbp-events2014-bin/target/appassembler/bin/keepOnlyBestResponses $KBPOPENREPO/params/interim_2015/generated/keepBest_${sysId}.params > $LOG.keepBest_${sysId}.log
      fi
    done
fi


{'fraction_non_alphanumeric': 0.10848585690515807, 'fraction_numerical': 0.02529118136439268, 'mean_word_length': 4.893137254901961, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5679262', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2138, 'n_words': 508}