<commit_before>#!/bin/bash

# Remove old backups


. "$(dirname "$0")/tarsnap-env.sh"


# Using tail to find archives to delete, but its
# +n syntax is out by one from what we want to do
# (also +0 == +1, so we're safe :-)
DAILY=`expr $DAILY + 1`
WEEKLY=`expr $WEEKLY + 1`
MONTHLY=`expr $MONTHLY + 1`
DATETIME_REGEX="[[:digit:]]{8}-[[:digit:]]{6}"


TMPFILE=/tmp/tarsnap.archives.$$
$TARSNAP --list-archives > $TMPFILE


delete() {
    DIR=$1
    TYPE=$2
    KEEP=$3
    for i in $(grep -E "^$DATETIME_REGEX-$TYPE-$DIR" $TMPFILE |
               sort -rn |
               tail -n +$KEEP)
    do
        echo "Deleting $i..."
        $TARSNAP --cachedir $CACHE_DIR -d -f $i
    done
}


# Do deletes
for dir in $DIRS; do
    delete $dir daily $DAILY
    delete $dir weekly $WEEKLY
    delete $dir monthly $MONTHLY
done


# Look for unrecognized backups

echoerr() {
    echo "$@" 1>&2
}

WEIRD=$(grep -Ev "^$DATETIME_REGEX-(daily|weekly|monthly)-(${DIRS// /|})" $TMPFILE)
if [ -n "$WEIRD" ]
then
    echoerr "Unrecognized backup(s) found:"
    for i in $WEIRD
    do
        echoerr " - $i" >&2
    done
fi

<commit_msg>set -e is weird sometimes<commit_after>#!/bin/bash

# Remove old backups


. "$(dirname "$0")/tarsnap-env.sh"


# Using tail to find archives to delete, but its
# +n syntax is out by one from what we want to do
# (also +0 == +1, so we're safe :-)
DAILY=`expr $DAILY + 1`
WEEKLY=`expr $WEEKLY + 1`
MONTHLY=`expr $MONTHLY + 1`
DATETIME_REGEX="[[:digit:]]{8}-[[:digit:]]{6}"


TMPFILE=/tmp/tarsnap.archives.$$
$TARSNAP --list-archives > $TMPFILE


delete() {
    DIR=$1
    TYPE=$2
    KEEP=$3
    for i in $(grep -E "^$DATETIME_REGEX-$TYPE-$DIR" $TMPFILE |
               sort -rn |
               tail -n +$KEEP)
    do
        echo "Deleting $i..."
        $TARSNAP --cachedir $CACHE_DIR -d -f $i
    done
}


# Do deletes
for dir in $DIRS; do
    delete $dir daily $DAILY
    delete $dir weekly $WEEKLY
    delete $dir monthly $MONTHLY
done


# Look for unrecognized backups

echoerr() {
    echo "$@" 1>&2
}

# `true` prevents exit on error
WEIRD=$(grep -Ev "^$DATETIME_REGEX-(daily|weekly|monthly)-(${DIRS// /|})" $TMPFILE || true)

if [ -n "$WEIRD" ]
then
    echoerr "Unrecognized backup(s) found:"
    for i in $WEIRD
    do
        echoerr " - $i" >&2
    done
fi


{'fraction_non_alphanumeric': 0.16673941510257528, 'fraction_numerical': 0.012221737232649499, 'mean_word_length': 2.6208530805687205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2227519', 'n_tokens_mistral': 976, 'n_tokens_neox': 890, 'n_words': 256}