<commit_before>#! /bin/bash
#############################################################################
#Runs one observation at a time in grid engine.  Second level program for 
#running firstpass on the MIT machines. First level program is 
#batch_firstpass.sh
#############################################################################

#$ -V
#$ -N firstpass
#$ -S /bin/bash

# obs_id, outdir, version, and thresh and nslots expected to be passed from qsub call
# in batch_firstpass.sh

echo JOBID ${JOB_ID}
echo TASKID ${SGE_TASK_ID}
obs_id=$(pull_args.py $*)
echo OBSID ${obs_id}

ls $outdir > /dev/null # ping the output directory so nfs automounts

# TODO: check wedge stats and stop if above thresh
if [ "${thresh}" -gt "0" ]; then
    # check db for wedgie
    echo Checking wedgie
    export PGPASSWORD=<PASSWORD>	
    # first check that there is an entry
    exists=`psql -h eor-00 mwa_qc -U mwa -c "select count(*) from qs where obsid=${obs_id};" -t -A`
    if [ $exists -eq 1 ]; then
	under=`psql -h eor-00 mwa_qc -U mwa -c "select count(*) from qs where obsid=${obs_id} and window_x<${thresh} and window_y<${thresh};" -t -A`
	if [ $under -eq 0 ]; then
	    echo Obs $obs_id did not meet wedge threshold. Skipping!
	    status_file=${outdir}/fhd_${version}/metdata/${obs_id}_status.txt
	    echo 'WEDGE_FLAGGED 1' > ${status_file}
	    echo 'COMPLETE 1' >> ${status_file}
	    exit 1
	else
	    echo Obs $obs_id met wedge criteria. Proceeding.
	fi
    else
	echo Did not find wedge stats for obs ${obs_id}. Proceeding.
    fi
fi

ls $outdir > /dev/null # ping the output directory so nfs automounts
/usr/local/bin/idl -IDL_DEVICE ps -IDL_CPU_TPOOL_NTHREADS $nslots -e eor_firstpass_versions -args $obs_id $outdir $version 

if [ $? -eq 0 ]
then
    echo "Finished"
    exit 0
else
    echo "Job Failed"
    exit 1
fi

<commit_msg>fixed bug in firstpass script<commit_after>#! /bin/bash
#############################################################################
#Runs one observation at a time in grid engine.  Second level program for 
#running firstpass on the MIT machines. First level program is 
#batch_firstpass.sh
#############################################################################

#$ -V
#$ -N firstpass
#$ -S /bin/bash

# obs_id, outdir, version, and thresh and nslots expected to be passed from qsub call
# in batch_firstpass.sh

echo JOBID ${JOB_ID}
echo TASKID ${SGE_TASK_ID}
obs_id=$(pull_args.py $*)
echo OBSID ${obs_id}

ls $outdir > /dev/null # ping the output directory so nfs automounts

# TODO: check wedge stats and stop if above thresh
if [ "${thresh}" -gt "0" ]; then
    # check db for wedgie
    echo Checking wedgie
    export PGPASSWORD=<PASSWORD>	
    # first check that there is an entry
    exists=`psql -h eor-00 mwa_qc -U mwa -c "select count(*) from qs where obsid=${obs_id};" -t -A`
    if [ $exists -eq 1 ]; then
	under=`psql -h eor-00 mwa_qc -U mwa -c "select count(*) from qs where obsid=${obs_id} and window_x<${thresh} and window_y<${thresh};" -t -A`
	if [ $under -eq 0 ]; then
	    echo Obs $obs_id did not meet wedge threshold. Skipping!
	    status_file=${outdir}/fhd_${version}/metdata/${obs_id}_status.txt
	    touch $status_file
	    echo 'WEDGE_FLAGGED 1' > ${status_file}
	    echo 'COMPLETE 1' >> ${status_file}
	    exit 1
	else
	    echo Obs $obs_id met wedge criteria. Proceeding.
	fi
    else
	echo Did not find wedge stats for obs ${obs_id}. Proceeding.
    fi
fi

ls $outdir > /dev/null # ping the output directory so nfs automounts
/usr/local/bin/idl -IDL_DEVICE ps -IDL_CPU_TPOOL_NTHREADS $nslots -e eor_firstpass_versions -args $obs_id $outdir $version 

if [ $? -eq 0 ]
then
    echo "Finished"
    exit 0
else
    echo "Job Failed"
    exit 1
fi


{'fraction_non_alphanumeric': 0.1839850026780932, 'fraction_numerical': 0.006963042313872523, 'mean_word_length': 3.715909090909091, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7638224', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1273, 'n_words': 444}