<commit_before>#! /bin/bash

# This script gets run on neuron-0-X via ssh from oven's ssh-master.py

# Don't write things to stdout - we pipe it to "tar" on the other end.

BACKEND_CLIENT="python /data1/dstn/dsolver/astrometry/blind/backend_client.py"
JOBDIR=/data1/dstn/dsolver/jobs/

# Read jobid
read -s jobid

# Replace "-" by "/" in the jobid.
jobid=`echo "$jobid" | sed s+-+/+g`

# Create our job directory.
cd $JOBDIR
mkdir -p $jobid
cd $jobid
mkdir `hostname -s`
cd `hostname -s`

# Read input file length and contents.
read -s nbytes
echo "Will read $nbytes bytes..." 1>&2
dd bs=1 count=$nbytes of=job.axy

# stderr goes back over the ssh tunnel...

$BACKEND_CLIENT `pwd`/job.axy `pwd`/../cancel > backend.stdout &

while [ 1 ]; do
    # Wait for a command from the master, with 1-second timeout...
    #echo "Reading..."
    read -t 1 command
    #echo "Got command: $command"
    if [ x$command != x ]; then
	echo "Got command: $command"
	echo "Killing job..."
	kill %%
	echo "Waiting..."
	wait
	break;
    fi
    # Check if the process finished.
    jobs %% > /dev/null
    jobstat=$?
    #echo "jobs command returned: $jobstat"
    if [ $jobstat -ne 0 ]; then
	break;
    fi
done

# Send back all the files we generated
tar cf - --ignore-failed-read --exclude=job.axy * ../solved<commit_msg>shard: wait for 'cancel' command; write cancel file.<commit_after>#! /bin/bash

# This script gets run on neuron-0-X via ssh from oven's ssh-master.py

# Don't write things to stdout - we pipe it to "tar" on the other end.

BACKEND_CLIENT="python /data1/dstn/dsolver/astrometry/blind/backend_client.py"
JOBDIR=/data1/dstn/dsolver/jobs/

# Read jobid
read -s jobid

# Replace "-" by "/" in the jobid.
jobid=`echo "$jobid" | sed s+-+/+g`

# Create our job directory.
cd $JOBDIR
mkdir -p $jobid
cd $jobid
mkdir `hostname -s`
cd `hostname -s`

# Read input file length and contents.
read -s nbytes
echo "Will read $nbytes bytes..." 1>&2
dd bs=1 count=$nbytes of=job.axy

# stderr goes back over the ssh tunnel...

$BACKEND_CLIENT `pwd`/job.axy `pwd`/../cancel > backend.stdout &

while [ 1 ]; do
    # Wait for a command from the master, with 1-second timeout...
    read -t 1 command
    if [ x$command != x ]; then
	echo "Got command: $command" 1>&2
	#echo "Killing job..."
	#kill %%
	echo "Cancelling job..." 1>&2
	touch `pwd`/../cancel
	echo "Waiting..." 1>&2
	wait
	break;
    fi
    # Check if the process finished.
    jobs %% > /dev/null 2>/dev/null
    jobstat=$?
    if [ $jobstat -ne 0 ]; then
	echo "Job finished." 1>&2
	break;
    fi
done

# Send back all the files we generated
tar cf - --ignore-failed-read --exclude=job.axy * ../solved
{'fraction_non_alphanumeric': 0.14334213504337986, 'fraction_numerical': 0.01093926820067899, 'mean_word_length': 2.9405646359583955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '900182', 'n_tokens_mistral': 1001, 'n_tokens_neox': 950, 'n_words': 336}