<commit_before>#!/bin/bash
#
# Check for newly arrived newspaper batch(es), and initiate creation in DOMS
#

CONFIG_FILENAME="$1"
if [ -z "$CONFIG_FILENAME" ]; then
	echo 'config-filename not received' >&2
	exit 1
fi

# Get config from file
source "$CONFIG_FILENAME"

cd "$PATH_TO_DIR_OF_BATCHES"

for batch_dirname in *; do
	# Check format of dirname
	if [[ ! "$batch_dirname" =~ ^B[^-]+\-RT[0-9]+$ ]]; then
	#if [[ "$batch_dirname" =~ "^B[^-]+\-RT[0-9]+$" ]]; then
		# Dirname not recognized as a batch, skip it
		continue
	fi
	echo "'$batch_dirname'"

	batch_id=`echo "$batch_dirname" | sed -r 's/^B([^-]+).+/\1/g'`
	roundtrip=`echo "$batch_dirname" | sed -r 's/^B[^-]+\-RT([0-9]+)/\1/g'`
done
<commit_msg>trigger now does not re-process batches that have already been processed<commit_after>#!/bin/bash
#
# Check for newly arrived newspaper batch(es), and initiate creation in DOMS
#

donedir='batches-done'

config_filename="$1"
if [ -z "$config_filename" ]; then
	echo 'config-filename not received' >&2
	exit 1
fi

# Get config from file
source "$config_filename"

# Get path to directory of this script
my_path="`dirname \"$0\"`"              # relative
my_path="`( cd \"$my_path\" && pwd )`"  # absolutized and normalized
if [ -z "$my_path" ] ; then
	echo 'could not access path' >&2
    exit 1
fi

cd "$path_to_dir_of_batches"

for batch_dirname in *; do
	# Check format of dirname
	if [[ ! "$batch_dirname" =~ ^B[^-]+\-RT[0-9]+$ ]]; then
		# Dirname not recognized as a batch, skip it
		continue
	fi

	# Skip batches that are already done
	if [ -f "$my_path/$donedir/$batch_dirname" ]; then
		continue
	fi

	batch_id=`echo "$batch_dirname" | sed -r 's/^B([^-]+).+/\1/g'`
	roundtrip=`echo "$batch_dirname" | sed -r 's/^B[^-]+\-RT([0-9]+)/\1/g'`

	# Create batch in DOMS
	# TODO
	echo $batch_id
	echo $roundtrip

	# Mark batch as done, by creating an empty file with the batch's name
	touch "$my_path/$donedir/$batch_dirname"
done

{'fraction_non_alphanumeric': 0.17216494845360825, 'fraction_numerical': 0.011855670103092783, 'mean_word_length': 3.9015151515151514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4107676', 'n_tokens_mistral': 762, 'n_tokens_neox': 727, 'n_words': 215}