<commit_before>#!/bin/bash
#
# example script to start a task n times in parallel
# syntax: `dirname $0 <command> <numrepetitions>

mytest(){
  echo "job $TASKID/$TASKNUM started"
  sleep $(( $RANDOM % 10 ))
  echo "job $TASKID/$TASKNUM finished"
  return 1
}

dead=false
quit(){
  dead=true
  jobs -rl | awk '{print $2}' | xargs kill &>/dev/null
}

trap quit INT
trap quit KILL
trap quit TERM

cmd="$1"
[ -z "$cmd" ] && cmd=mytest && echo "no first argument given. cmd=mytest (test mode)" >&2
ncpus=`grep proc /proc/cpuinfo|wc -l`
num="$2"
[ -z "$num" ] && num=-1 && echo "no second argument given. num=-1 (infinite loop)" >&2

export TASKNUM=$num<commit_msg>parallel.sh: get job ids from jobs command<commit_after>#!/bin/bash
#
# example script to start a task n times in parallel
# syntax: `dirname $0 <command> <numrepetitions>

mytest(){
  echo "job $TASKID/$TASKNUM started"
  sleep $(( $RANDOM % 10 ))
  echo "job $TASKID/$TASKNUM finished"
  return 1
}

dead=false
quit(){
  dead=true
  kill `jobs -p`
}

trap quit INT
trap quit KILL
trap quit TERM

cmd="$1"
[ -z "$cmd" ] && cmd=mytest && echo "no first argument given. cmd=mytest (test mode)" >&2
ncpus=`grep proc /proc/cpuinfo|wc -l`
num="$2"
[ -z "$num" ] && num=-1 && echo "no second argument given. num=-1 (infinite loop)" >&2

export TASKNUM=$num
{'fraction_non_alphanumeric': 0.16704805491990846, 'fraction_numerical': 0.016018306636155607, 'mean_word_length': 3.988593155893536, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5675339', 'n_tokens_mistral': 514, 'n_tokens_neox': 504, 'n_words': 164}