<commit_before>#!/bin/bash
#
# A script to put different kind of loads on our AWS installation, to feed some
# numbers to Grafana.

# Assumes we have set things up with okapi-examples, or similar
#OKAPI="http://localhost:9130"
OKAPI="http://okapi-test.aws.indexdata.com:9130"
#OKAPI="http://okapi-test1.aws.indexdata.com:9130"

MODULE=${1:-'testb'}  # module to invoke, default: okapi-test-module (testb)
TENANT=${2:-'our'}     # tenant to invoke for
SLEEP=${3:-1}       # delay between invocations (sec)
TOKEN=${4:-"X-Okapi-Token: <PASSWORD>:peter:04415268d4170e95ec497077ad4cba3c"}
#echo " M='$MODULE' T='$TENANT' S='$SLEEP' T='$TOKEN'"

# Stop file - touch this to stop all instances of this script
STOP=aws.stop
rm -f $STOP

while [ ! -f $STOP ]
do

  echo
  date
  curl  -w '\n' -D -  \
      -H "X-Okapi-Tenant: $TENANT" \
      -H "$TOKEN" \
      $OKAPI/$MODULE

  sleep $SLEEP

done
<commit_msg>load-aws.sh with the new example data<commit_after>#!/bin/bash
#
# A script to put different kind of loads on our AWS installation, to feed some
# numbers to Grafana.

# Assumes we have set things up with okapi-examples, or similar
#OKAPI="http://localhost:9130"
OKAPI="http://okapi-test.aws.indexdata.com:9130"
#OKAPI="http://okapi-test1.aws.indexdata.com:9130"

MODULE=${1:-'testb'}  # module to invoke, default: okapi-test-module (testb)
TENANT=${2:-'testlib'}     # tenant to invoke for
SLEEP=${3:-1}       # delay between invocations (sec)
TOKEN=${4:-"X-Okapi-Token: <PASSWORD>:<PASSWORD>"}
#echo " M='$MODULE' T='$TENANT' S='$SLEEP' T='$TOKEN'"

# Stop file - touch this to stop all instances of this script
STOP=aws.stop
rm -f $STOP

while [ ! -f $STOP ]
do

  echo
  date
  curl  -w '\n' -D -  \
      -H "X-Okapi-Tenant: $TENANT" \
      -H "$TOKEN" \
      $OKAPI/$MODULE

  sleep $SLEEP

done

{'fraction_non_alphanumeric': 0.16537610619469026, 'fraction_numerical': 0.032079646017699116, 'mean_word_length': 3.5452261306532664, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897540', 'n_tokens_mistral': 723, 'n_tokens_neox': 695, 'n_words': 185}