<commit_before>#! /bin/bash

# For modules loaded, see "bashrc" in this directory.

export PYTHONPATH=${PYTHONPATH}:.

# Force MKL single-threaded
# https://software.intel.com/en-us/articles/using-threaded-intel-mkl-in-multi-thread-application
export MKL_NUM_THREADS=1

# Try limiting memory to avoid killing the whole MPI job...
#ulimit -S -v 15000000
ulimit -S -v 30000000
ulimit -a

outdir=$SCRATCH/dr3

brick="$1"

logdir=$(echo $brick | head -c 3)
mkdir -p $outdir/logs/$logdir
log="$outdir/logs/$logdir/$brick.log"

echo Logging to: $log
echo Running on ${NERSC_HOST} $(hostname)

echo -e "\n\n\n" >> $log
echo "-----------------------------------------------------------------------------------------" >> $log
echo "PWD: $(pwd)" >> $log
echo "Modules:" >> $log
module list >> $log 2>&1
echo >> $log
echo "Environment:" >> $log
set | grep -v PASS >> $log
echo >> $log
ulimit -a >> $log
echo >> $log

echo -e "\nStarting on ${NERSC_HOST} $(hostname)\n" >> $log
echo "-----------------------------------------------------------------------------------------" >> $log

# -s tims \
# -P 'pickles/runbrick-dr2p-%(brick)s-%%(stage)s.pickle' \
#    --no-early-coadds \
python legacypipe/runbrick.py \
     --no-write \
     --pipe \
     --threads 8 \
     --on-bricks \
     --skip \
     --skip-calibs \
     --checkpoint checkpoints/checkpoint-${brick}.pickle \
     --fitblobs-prereq pickles/runbrick-${brick}-srcs.pickle \
     --ps ps/ps-${brick}.fits \
     --brick $brick --outdir $outdir --nsigma 6 >> $log 2>&1

# qdo launch dr2n 16 --cores_per_worker 8 --walltime=24:00:00 --script ../bin/pipebrick.sh --batchqueue regular --verbose
# qdo launch edr0 4 --cores_per_worker 8 --batchqueue regular --walltime 4:00:00 --script ../bin/pipebrick.sh --keep_env --batchopts "--qos=premium -a 0-3"<commit_msg>allow missing brickq 0 now<commit_after>#! /bin/bash

# For modules loaded, see "bashrc" in this directory.

export PYTHONPATH=${PYTHONPATH}:.

# Force MKL single-threaded
# https://software.intel.com/en-us/articles/using-threaded-intel-mkl-in-multi-thread-application
export MKL_NUM_THREADS=1

# Try limiting memory to avoid killing the whole MPI job...
#ulimit -S -v 15000000
ulimit -S -v 30000000
ulimit -a

outdir=$SCRATCH/dr3

brick="$1"

logdir=$(echo $brick | head -c 3)
mkdir -p $outdir/logs/$logdir
log="$outdir/logs/$logdir/$brick.log"

echo Logging to: $log
echo Running on ${NERSC_HOST} $(hostname)

echo -e "\n\n\n" >> $log
echo "-----------------------------------------------------------------------------------------" >> $log
echo "PWD: $(pwd)" >> $log
echo "Modules:" >> $log
module list >> $log 2>&1
echo >> $log
echo "Environment:" >> $log
set | grep -v PASS >> $log
echo >> $log
ulimit -a >> $log
echo >> $log

echo -e "\nStarting on ${NERSC_HOST} $(hostname)\n" >> $log
echo "-----------------------------------------------------------------------------------------" >> $log

# -s tims \
# -P 'pickles/runbrick-dr2p-%(brick)s-%%(stage)s.pickle' \
#    --no-early-coadds \
python legacypipe/runbrick.py \
     --no-write \
     --pipe \
     --threads 8 \
     --on-bricks \
     --skip \
     --skip-calibs \
     --checkpoint checkpoints/checkpoint-${brick}.pickle \
     --fitblobs-prereq pickles/runbrick-${brick}-srcs.pickle \
     --ps ps/ps-${brick}.fits \
     --brick $brick --outdir $outdir --nsigma 6 \
     --allow-missing-brickq 0 >> $log 2>&1

# qdo launch dr2n 16 --cores_per_worker 8 --walltime=24:00:00 --script ../bin/pipebrick.sh --batchqueue regular --verbose
# qdo launch edr0 4 --cores_per_worker 8 --batchqueue regular --walltime 4:00:00 --script ../bin/pipebrick.sh --keep_env --batchopts "--qos=premium -a 0-3"
{'fraction_non_alphanumeric': 0.2622100954979536, 'fraction_numerical': 0.026193724420190995, 'mean_word_length': 2.080672268907563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '895034', 'n_tokens_mistral': 1378, 'n_tokens_neox': 1279, 'n_words': 321}