<commit_before>#!/bin/sh
###############################################################################
#
# Setup script for all OSSIM repositories
#
# The test data directory, specified by the env var OSSIM_BATCH_TEST_DATA is 
# syncronized against a master repository.
# 
# OSSIM_BATCH_TEST_DATAmust be defined prior.
#
###############################################################################
# Sync against S3 for test data:
if [ -z $OSSIM_DATA ] || [ ! -d $OSSIM_DATA ] ; then
  echo "ERROR: Env var OSSIM_DATA=$OSSIM_DATA does not point to a valid data directory. Aborting setup..."; 
  exit 1
fi

echo; echo "STATUS: Syncing data directory to S3..."
s3cmd sync --no-check-md5 s3://yumrepos-dev-rbtcloud/ossim_data/public $OSSIM_DATA/
if [ $? != 0 ]; then
  echo "ERROR: Failed S3 sync."
  exit 1
fi
echo "STATUS: S3 sync successful..."

exit 0
<commit_msg>use locally shared data directory<commit_after>#!/bin/sh
###############################################################################
#
# Setup script for all OSSIM repositories
#
# The test data directory, specified by the env var OSSIM_BATCH_TEST_DATA is
# syncronized against a master repository.
#
# OSSIM_BATCH_TEST_DATAmust be defined prior.
#
###############################################################################
# Sync against S3 for test data:
if [ -z $OSSIM_DATA ] || [ ! -d $OSSIM_DATA ] ; then
  echo "ERROR: Env var OSSIM_DATA=$OSSIM_DATA does not point to a valid data directory. Aborting setup...";
  exit 1
fi

exit 0
{'fraction_non_alphanumeric': 0.28825857519788917, 'fraction_numerical': 0.009234828496042216, 'mean_word_length': 5.1417004048583, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6237237', 'n_tokens_mistral': 470, 'n_tokens_neox': 438, 'n_words': 151}