<commit_before>#!/bin/bash

set -e

<%
  require "cgi"
  require "shellwords"
%>

RUN_DIR=/var/vcap/sys/run/groundcrew
LOG_DIR=/var/vcap/sys/log/groundcrew
PIDFILE=$RUN_DIR/beacon.pid

source /var/vcap/packages/pid_utils/pid_utils.sh

kill_session_siblings_of() {
  pidfile=$1

  if [ -f "$pidfile" ]; then
    pid=$(head -1 "$pidfile")

    session=$(ps -p $pid -o sid=)
    ps -s $session -o pid:1= | xargs kill

    rm -f $pidfile
  else
    echo "Pidfile $pidfile doesn't exist!"
  fi
}

case $1 in

  start)
    pid_guard $PIDFILE "beacon"

    mkdir -p $RUN_DIR
    chown -R vcap:vcap $RUN_DIR

    mkdir -p $LOG_DIR
    chown -R vcap:vcap $LOG_DIR

    echo $$ > $PIDFILE

    for key in /var/vcap/packages/generated_worker_key/id_rsa \
               /var/vcap/jobs/groundcrew/config/private_key; do
      chown vcap:vcap $key
      chmod 0400 $key
    done

    {
      echo -n '[<%= p("groundcrew.tsa.host") %>]:<%= p("groundcrew.tsa.port") %> '
      cat /var/vcap/packages/generated_tsa_host_key/id_rsa.pub
    } > /var/vcap/jobs/groundcrew/config/generated_tsa_known_hosts

    KNOWN_HOSTS_FILE=${RUN_DIR}/known_hosts
    <% if p("groundcrew.tsa.host_public_key") != "" %>
    cp /var/vcap/jobs/groundcrew/config/known_hosts $KNOWN_HOSTS_FILE
    <% else %>
    cp /var/vcap/jobs/groundcrew/config/generated_tsa_known_hosts $KNOWN_HOSTS_FILE
    <% end %>

    chown vcap:vcap $KNOWN_HOSTS_FILE

    cat /var/vcap/jobs/groundcrew/config/worker.json | \
      sed -e "s/HOSTNAME_REPLACE_ME/$(hostname)/g" > \
      $RUN_DIR/worker.json

    while true; do
      chpst -u vcap:vcap ssh \
        -p <%= p("groundcrew.tsa.port") %> <%= p("groundcrew.tsa.host") %> \
        <% if p("groundcrew.tsa.private_key") != "" %> \
        -i /var/vcap/jobs/groundcrew/config/private_key \
        <% else %> \
        -i /var/vcap/packages/generated_worker_key/id_rsa \
        <% end %> \
        -o UserKnownHostsFile=${KNOWN_HOSTS_FILE} \
        -o ConnectTimeout=30 \
        -o ServerAliveInterval=10 \
        -o ServerAliveCountMax=3 \
        <% if p("groundcrew.garden.forward_address", nil) %> \
          -R0.0.0.0:7777:<%= p("groundcrew.garden.forward_address") %> \
          <% if p("groundcrew.baggageclaim.forward_address", nil) %> \
          -R0.0.0.0:7788:<%= p("groundcrew.baggageclaim.forward_address") %> \
          <% end %> \
          forward-worker \
          --garden 0.0.0.0:7777 \
          <% if p("groundcrew.baggageclaim.forward_address", nil) %> \
          --baggageclaim 0.0.0.0:7788 \
          <% end %> \
        <% else %> \
          register-worker \
        <% end %> \
        < $RUN_DIR/worker.json \
        1>>$LOG_DIR/beacon.stdout.log \
        2>>$LOG_DIR/beacon.stderr.log || \
        true

      sleep 1
    done

    ;;

  stop)
    kill_session_siblings_of $PIDFILE

    ;;

  *)
    echo "Usage: $0 {start|stop}"

    ;;

esac
<commit_msg>tweak the ssh keepalive interval<commit_after>#!/bin/bash

set -e

<%
  require "cgi"
  require "shellwords"
%>

RUN_DIR=/var/vcap/sys/run/groundcrew
LOG_DIR=/var/vcap/sys/log/groundcrew
PIDFILE=$RUN_DIR/beacon.pid

source /var/vcap/packages/pid_utils/pid_utils.sh

kill_session_siblings_of() {
  pidfile=$1

  if [ -f "$pidfile" ]; then
    pid=$(head -1 "$pidfile")

    session=$(ps -p $pid -o sid=)
    ps -s $session -o pid:1= | xargs kill

    rm -f $pidfile
  else
    echo "Pidfile $pidfile doesn't exist!"
  fi
}

case $1 in

  start)
    pid_guard $PIDFILE "beacon"

    mkdir -p $RUN_DIR
    chown -R vcap:vcap $RUN_DIR

    mkdir -p $LOG_DIR
    chown -R vcap:vcap $LOG_DIR

    echo $$ > $PIDFILE

    for key in /var/vcap/packages/generated_worker_key/id_rsa \
               /var/vcap/jobs/groundcrew/config/private_key; do
      chown vcap:vcap $key
      chmod 0400 $key
    done

    {
      echo -n '[<%= p("groundcrew.tsa.host") %>]:<%= p("groundcrew.tsa.port") %> '
      cat /var/vcap/packages/generated_tsa_host_key/id_rsa.pub
    } > /var/vcap/jobs/groundcrew/config/generated_tsa_known_hosts

    KNOWN_HOSTS_FILE=${RUN_DIR}/known_hosts
    <% if p("groundcrew.tsa.host_public_key") != "" %>
    cp /var/vcap/jobs/groundcrew/config/known_hosts $KNOWN_HOSTS_FILE
    <% else %>
    cp /var/vcap/jobs/groundcrew/config/generated_tsa_known_hosts $KNOWN_HOSTS_FILE
    <% end %>

    chown vcap:vcap $KNOWN_HOSTS_FILE

    cat /var/vcap/jobs/groundcrew/config/worker.json | \
      sed -e "s/HOSTNAME_REPLACE_ME/$(hostname)/g" > \
      $RUN_DIR/worker.json

    while true; do
      chpst -u vcap:vcap ssh \
        -p <%= p("groundcrew.tsa.port") %> <%= p("groundcrew.tsa.host") %> \
        <% if p("groundcrew.tsa.private_key") != "" %> \
        -i /var/vcap/jobs/groundcrew/config/private_key \
        <% else %> \
        -i /var/vcap/packages/generated_worker_key/id_rsa \
        <% end %> \
        -o UserKnownHostsFile=${KNOWN_HOSTS_FILE} \
        -o ConnectTimeout=30 \
        -o ServerAliveInterval=8 \
        -o ServerAliveCountMax=3 \
        <% if p("groundcrew.garden.forward_address", nil) %> \
          -R0.0.0.0:7777:<%= p("groundcrew.garden.forward_address") %> \
          <% if p("groundcrew.baggageclaim.forward_address", nil) %> \
          -R0.0.0.0:7788:<%= p("groundcrew.baggageclaim.forward_address") %> \
          <% end %> \
          forward-worker \
          --garden 0.0.0.0:7777 \
          <% if p("groundcrew.baggageclaim.forward_address", nil) %> \
          --baggageclaim 0.0.0.0:7788 \
          <% end %> \
        <% else %> \
          register-worker \
        <% end %> \
        < $RUN_DIR/worker.json \
        1>>$LOG_DIR/beacon.stdout.log \
        2>>$LOG_DIR/beacon.stderr.log || \
        true

      sleep 1
    done

    ;;

  stop)
    kill_session_siblings_of $PIDFILE

    ;;

  *)
    echo "Usage: $0 {start|stop}"

    ;;

esac

{'fraction_non_alphanumeric': 0.15646492434662998, 'fraction_numerical': 0.016678129298486933, 'mean_word_length': 2.2084942084942085, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3386669', 'n_tokens_mistral': 2438, 'n_tokens_neox': 2215, 'n_words': 344}