<commit_before>#!/bin/bash
# Based on 15 minutely backups
MAX_BACKUPS=$((60/15 * 24 * 7))
BACKUP=$(date +%Y-%m-%d_%H:%M)
INC=''
BACKUP_PREVIOUS=''
cd "$(dirname "$(readlink -f "$0")")"
mkdir -p backups

set_rsync_opts() {
	BACKUP_PREVIOUS=$(find backups -maxdepth 1 | sort -n | tail -1)

	if [[ $BACKUP_PREVIOUS ]]; then
		inc=--link-dest="$PWD/$BACKUP_PREVIOUS"
	fi
}

cleanup() {
	local ALL_BACKUPS
	local BACKUPS_TO_REMOVE

	ALL_BACKUPS="$(find backups -maxdepth 1 | tail -n +2)"
	BACKUPS_TO_REMOVE="$(echo "$ALL_BACKUPS" | tail -n +$MAX_BACKUPS)"

	for i in $BACKUPS_TO_REMOVE; do
		echo rm -rf $BACKUPS_TO_REMOVE
	done
}

set_rsync_opts
mkdir -p backups/$BACKUP
rsync -aHAX $inc data/ backups/$BACKUP/
cleanup
<commit_msg>Only keep 2 days of backups<commit_after>#!/bin/bash
# Based on 15 minutely backups
MAX_BACKUPS=$((60/15 * 24 * 2))
BACKUP=$(date +%Y-%m-%d_%H:%M)
INC=''
BACKUP_PREVIOUS=''
cd "$(dirname "$(readlink -f "$0")")"
mkdir -p backups

set_rsync_opts() {
	BACKUP_PREVIOUS=$(find backups -maxdepth 1 | sort -n | tail -1)

	if [[ $BACKUP_PREVIOUS ]]; then
		inc=--link-dest="$PWD/$BACKUP_PREVIOUS"
	fi
}

cleanup() {
	local ALL_BACKUPS
	local BACKUPS_TO_REMOVE

	ALL_BACKUPS="$(find backups -maxdepth 1 | tail -n +2)"
	BACKUPS_TO_REMOVE="$(echo "$ALL_BACKUPS" | tail -n +$MAX_BACKUPS)"

	for i in $BACKUPS_TO_REMOVE; do
		rm -rf $BACKUPS_TO_REMOVE
	done
}

set_rsync_opts
mkdir -p backups/$BACKUP
rsync -aHAX $inc data/ backups/$BACKUP/
cleanup

{'fraction_non_alphanumeric': 0.16404647983595352, 'fraction_numerical': 0.019822282980177717, 'mean_word_length': 4.609195402298851, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3155051', 'n_tokens_mistral': 687, 'n_tokens_neox': 629, 'n_words': 148}