<commit_before>#!/bin/bash -eux
if [ -f /tmp/.running ]
then
    echo "Already running!!!"
    exit 0
fi

touch /tmp/.running
trap "rm /tmp/.running ; sleep 10 " EXIT

s3_dir=${AWS_S3_LOCAL_MOUNT_POINT}
local_dir=${SHARE_VOLUME}
lower=${SIMPLE_QUOTA_LOWER_MB}
upper=${SIMPLE_QUOTA_UPPER_MB}
max_files=${SIMPLE_QUOTA_MAX_FILES}

dir_size=$( du -sm "$local_dir" | cut -f1 )
echo "$local_dir usage is currently ${dir_size}MB limit is ${lower}MB (hard limit ${upper}) MB"

if [ ! -f ${local_dir}/.syncinit ]
then
    echo "Copying from S3"
    rsync -arv --delete-after --backup --backup-dir=${local_dir}/.syncbackrev  --exclude ".sync*"  ${s3_dir}/ ${local_dir}/
    touch ${local_dir}/.syncinit
fi

inotifywait -r ${local_dir}

changed=$(find $local_dir -newer ${local_dir}/.synclast -type f -not -path "${local_dir}/.sync*/*" | wc -l)


if (( $changed == 0 )) && [[ -f ${local_dir}/.synclast ]]
then
    echo "No changes"
    exit 0
fi

num_files=$(find $local_dir  -type f -not -path "${local_dir}/.sync*/*" | wc -l)


if (( ${num_files} > ${max_files} ))
then
    echo "Too many files: ${num_files} > ${max_files}"
fi

if (( ${dir_size} > ${lower} ))  ||  (( ${num_files} > ${max_files} ))
then
    if [ ! -f ${local_dir}/.sync-quota-exceeded ]
    then
        touch ${local_dir}/.sync-quota-exceeded
    fi
    rm -rf ${local_dir}/.syncbackup || true
else
    if [ -f ${local_dir}/.sync-quota-exceeded ]
    then
        rm ${local_dir}/.sync-quota-exceeded
    fi
    touch ${local_dir}/.synclast
    rsync -arv --delete-after --backup --backup-dir=${local_dir}/.syncbackup  --exclude ".sync*"   ${local_dir}/ ${s3_dir}/
fi

#When over upper quota, we delete the files created since we hit the lower quota
while [ $dir_size -gt ${upper} ] ; do
    find $local_dir -type f -newer ${local_dir}/.sync-quota-exceeded -exec rm -f {} \;
    find $local_dir -type d -empty -delete
done<commit_msg>And that's it folks, the bare bones of a continuously backed up volume.<commit_after>#!/bin/bash -eu
if [ -f /tmp/.running ]
then
    echo "Already running!!!"
    exit 0
fi

touch /tmp/.running
trap "rm /tmp/.running ; sleep 1 " EXIT

s3_dir=${AWS_S3_LOCAL_MOUNT_POINT}
local_dir=${SHARE_VOLUME}
lower=${SIMPLE_QUOTA_LOWER_MB}
upper=${SIMPLE_QUOTA_UPPER_MB}
max_files=${SIMPLE_QUOTA_MAX_FILES}

dir_size=$( du -sm "$local_dir" | cut -f1 )
echo "$local_dir usage is currently ${dir_size}MB limit is ${lower}MB (hard limit ${upper}MB)"

if [ ! -f ${local_dir}/.syncinit ]
then
    echo "Copying from S3"
    rsync -arv --delete-after --backup --backup-dir=${local_dir}/.syncbackrev  --exclude ".sync*"  ${s3_dir}/ ${local_dir}/
    touch ${local_dir}/.syncinit
fi

inotifywait -t 60 -r ${local_dir}

changed=$(find $local_dir -newer ${local_dir}/.synclast -type f -not -path "${local_dir}/.sync*/*" | wc -l)


if (( $changed == 0 )) && [[ -f ${local_dir}/.synclast ]]
then
    echo "No changes"
    exit 0
fi

num_files=$(find $local_dir  -type f -not -path "${local_dir}/.sync*/*" | wc -l)


if (( ${num_files} > ${max_files} ))
then
    echo "Too many files: ${num_files} > ${max_files}"
fi

if (( ${dir_size} > ${lower} ))  ||  (( ${num_files} > ${max_files} ))
then
    echo "Quota exceeded"
    if [ ! -f ${local_dir}/.sync-quota-exceeded ]
    then
        touch ${local_dir}/.sync-quota-exceeded
    fi
    rm -rf ${local_dir}/.syncbackup || true
else
    echo "Within quota"
    if [ -f ${local_dir}/.sync-quota-exceeded ]
    then
        rm ${local_dir}/.sync-quota-exceeded
    fi
    touch ${local_dir}/.synclast
    rsync -arv --delete-after --backup --backup-dir=${local_dir}/.syncbackup  --exclude ".sync*"   ${local_dir}/ ${s3_dir}/
fi

#When over upper quota, we delete the files created since we hit the lower quota
while [ $dir_size -gt ${upper} ] ; do
    echo "Purge files as upper limt exceeded"
    find $local_dir -type f -newer ${local_dir}/.sync-quota-exceeded -exec rm -f {} \;
    find $local_dir -type d -empty -delete
done
{'fraction_non_alphanumeric': 0.17490494296577946, 'fraction_numerical': 0.0058301647655259825, 'mean_word_length': 3.256742179072276, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5843757', 'n_tokens_mistral': 1534, 'n_tokens_neox': 1476, 'n_words': 391}