<commit_before><commit_msg>Added piper.sh for reference.<commit_after>#! /bin/bash
#
# TO TEST IT -- execute in separate shells
# rm hello ; nc -k -l -U hello 1>&0
# ./piper hello 12345
# nc localhost 12345

if [ $# -le 1 ]
then
  echo Usage: $0 socket port
  exit 1
fi

PIPE_IN="/tmp/piper-pipe-in"
PIPE_OUT="/tmp/piper-pipe-out"

# TODO only one piper script can run at the same time
# make these random
rm $PIPE_IN &> /dev/null
rm $PIPE_OUT &> /dev/null

mkfifo $PIPE_IN
mkfifo $PIPE_OUT

PID1=-1
PID2=-1
RUNNING=true

trap cleanup INT

function cleanup() {
  echo "trapped ctrl-c"
  RUNNING=""

  rm $PIPE_IN
  rm $PIPE_OUT

  kill -15 $PID1
  kill -15 $PID2
}

while [ $RUNNING ]; do
  nc -l 127.0.0.1 $2 > $PIPE_IN < $PIPE_OUT &
  PID1=$!

  nc -U $1 < $PIPE_IN > $PIPE_OUT &
  PID2=$!

  wait $PID1 $PID2 2> /dev/null
done

echo "finished"
{'fraction_non_alphanumeric': 0.13829787234042554, 'fraction_numerical': 0.04491725768321513, 'mean_word_length': 3.0526315789473686, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2506721', 'n_tokens_mistral': 402, 'n_tokens_neox': 362, 'n_words': 110}