<commit_before>#!/bin/bash
#
# Usage: start.sh [ <consumer_name> [-v] ]
#
# If starting a single consumer, you can pass -v as the 2nd argument
# to run in the foreground

# By default, run in the background (-d = detached)
FGBG=${2:--d}

function get_count
{
  COUNT_FILE=${1//.yaml/.qty}
  COUNT=$(cat $COUNT_FILE)
  echo $COUNT
}

function start_consumers
{
  CONSUMER=$(basename ${1} .yaml)

  if [ "${FGBG}" == "-v" ]; then
    CONSUMERS=1
  else
    CONSUMERS=$(get_count $1)
  fi

  echo "Starting $CONSUMERS $CONSUMER consumer(s)"
  for x in $(seq 1 $CONSUMERS); do
    sudo su rejected -c "/opt/rejected/bin/rejected.py -c $1 ${FGBG}"
  done

  if [ "${FGBG}" != "-v" ]; then
    /opt/rejected/bin/opslog.py "Started $CONSUMERS $CONSUMER rejected consumer(s)"
  fi
}

if [ -z "$1" ]; then
  for CONFIG in /opt/rejected/config/*yaml
  do
    start_consumers $CONFIG
  done
else
  for CONFIG in /opt/rejected/config/$1.yaml
  do
    start_consumers $CONFIG
  done
fi

sudo /opt/rejected/bin/setaffinity.sh 1> /dev/null<commit_msg>Change the startup -v flag to -f<commit_after>#!/bin/bash
#
# Usage: start.sh [ <consumer_name> [-f] ]
#
# If starting a single consumer, you can pass -f as the 2nd argument
# to run in the foreground

# By default, run in the background (-d = detached)
FGBG=${2:--d}

function get_count
{
  COUNT_FILE=${1//.yaml/.qty}
  COUNT=$(cat $COUNT_FILE)
  echo $COUNT
}

function start_consumers
{
  CONSUMER=$(basename ${1} .yaml)

  if [ "${FGBG}" == "-f" ]; then
    CONSUMERS=1
  else
    CONSUMERS=$(get_count $1)
  fi

  echo "Starting $CONSUMERS $CONSUMER consumer(s)"
  for x in $(seq 1 $CONSUMERS); do
    sudo su rejected -c "/opt/rejected/bin/rejected.py -c $1 ${FGBG}"
  done

  if [ "${FGBG}" != "-f" ]; then
    /opt/rejected/bin/opslog.py "Started $CONSUMERS $CONSUMER rejected consumer(s)"
  fi
}

if [ -z "$1" ]; then
  for CONFIG in /opt/rejected/config/*yaml
  do
    start_consumers $CONFIG
  done
else
  for CONFIG in /opt/rejected/config/$1.yaml
  do
    start_consumers $CONFIG
  done
fi

sudo /opt/rejected/bin/setaffinity.sh 1> /dev/null
{'fraction_non_alphanumeric': 0.15208034433285508, 'fraction_numerical': 0.010521281683405069, 'mean_word_length': 3.209255533199195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1243398', 'n_tokens_mistral': 845, 'n_tokens_neox': 790, 'n_words': 227}