<commit_before>#!/bin/bash

BACKUP_LIST=(
    "/var/www-prod/app/webroot/img/profiles_128"
    "/var/www-prod/app/webroot/img/profiles_36"
    "/home/debian/tatowiki/tatowiki/app"
    "/var/www-audio/sentences"
    "/home/debian/dump/db.sql.gz"
)

GDRIVE="/home/debian/gdrive/gdrive"

BACKUP_DIR="backupdir"
mkdir -p $BACKUP_DIR

# in Mbs
CHUNK_SIZE=32
CHUNK_SIZE=$(echo "$CHUNK_SIZE*1024*1024" | bc)

KEEP_NUM=2

DATE=$(date -u +%F)

prune(){
    search_name=$1
    keep_num=$2

    flst=$($GDRIVE list --query "name contains '$search_name'" --order "createdTime desc" | tail -n +2);
    curr_num=$(echo "$flst" | wc -l)
    if [[ $curr_num -gt $keep_num ]]; then
        dlst=$(diff <(echo "$flst") <(echo "$flst" | head -n $keep_num) |\
         tail -n +2 | sed 's/< //g' | awk '{print $1}'
        )

        echo "Pruning $(echo "$dlst" | wc -l) files related to $search_name ..."
        IFS=$'\n' dlst=($dlst)
        for d in ${dlst[@]}; do
            echo "Deleting $d"
            $GDRIVE delete "$d"
        done
    fi
}

echo "Starting gdrive backup script at $DATE ..."

for i in "${BACKUP_LIST[@]}"; do
    # check for file or dir
    if [[ "$(echo $i | grep -o '\.' | uniq)" == "." ]]; then
        f=$(echo "$i" | sed 's/^\///g; s/\//-/g')

        ext=$(echo "$f" | grep -Po "\..*")
        f=$(echo "$f" | sed s/\\..*$//g)

        search_name="$f"
        prune "$search_name" $(echo "$KEEP_NUM-1" | bc)

        f="$f""_""$DATE$ext"
        echo "Uploading file $f ..."
        $GDRIVE upload --name "$f" --chunksize "$CHUNK_SIZE" "$i"
    else
        f=$(echo "$i" | sed 's/^\///g; s/\//-/g')

        search_name="$f"
        prune "$search_name" $(echo "$KEEP_NUM-1" | bc)

        f="$f""_""$DATE.tar"
        echo "Tarring dir $i into $f ..."
        tar -cf "$BACKUP_DIR/$f" "$i"
        echo "Uploading Tarred dir file $f"
        $GDRIVE upload --delete --chunksize "$CHUNK_SIZE" "$BACKUP_DIR/$f"
    fi
done<commit_msg>gdrive backup script fixes: variable quoting, auth, file/dir checking<commit_after>#!/bin/bash

BACKUP_LIST=(
    "/var/www-prod/app/webroot/img/profiles_128"
    "/var/www-prod/app/webroot/img/profiles_36"
    "/home/debian/tatowiki/tatowiki/app"
    "/var/www-audio/sentences"
    "/home/debian/dump/db.sql.gz"
)

CONFIG_DIR="/home/debian/.gdrive/"
AUTH_JSON="tatoeba-backup.json"
GDRIVE_BIN="/home/debian/gdrive/gdrive"
GDRIVE(){
    $GDRIVE_BIN --config "$CONFIG_DIR" --service-account "$AUTH_JSON" "$@"
}

BACKUP_DIR="backupdir"
mkdir -p $BACKUP_DIR

# in Mbs
CHUNK_SIZE=32
CHUNK_SIZE=$(($CHUNK_SIZE*1024*1024))

KEEP_NUM=2

DATE=$(date -u +%F)

prune(){
    search_name=$1
    keep_num=$2

    flst=$(GDRIVE list --query "name contains '$search_name'" --order "createdTime desc" | tail -n +2);
    curr_num=$(echo "$flst" | wc -l)
    if [[ $curr_num -gt $keep_num ]]; then
        dlst=$(diff <(echo "$flst") <(echo "$flst" | head -n $keep_num) |\
         tail -n +2 | sed 's/< //g' | awk '{print $1}'
        )

        echo "Pruning $(echo "$dlst" | wc -l) files related to $search_name ..."
        IFS=$'\n' dlst=($dlst)
        for d in "${dlst[@]}"; do
            echo "Deleting $d"
            GDRIVE delete "$d"
        done
    fi
}

echo "Starting gdrive backup script at $DATE ..."

for i in "${BACKUP_LIST[@]}"; do
    # check for file or dir
    if [[ -f "$i" ]]; then
        f=$(echo "$i" | sed 's/^\///g; s/\//-/g')

        ext=$(echo "$f" | grep -Po "\..*")
        f=$(echo "$f" | sed s/\\..*$//g)

        search_name="$f"
        prune "$search_name" $(echo "$KEEP_NUM-1" | bc)

        f="$f""_""$DATE$ext"
        echo "Uploading file $f ..."
        GDRIVE upload --name "$f" --chunksize "$CHUNK_SIZE" "$i"
    elif [[ -d "$i" ]]; then
        f=$(echo "$i" | sed 's/^\///g; s/\//-/g')

        search_name="$f"
        prune "$search_name" $(echo "$KEEP_NUM-1" | bc)

        f="$f""_""$DATE.tar"
        echo "Tarring dir $i into $f ..."
        tar -cf "$BACKUP_DIR/$f" "$i"
        echo "Uploading Tarred dir file $f"
        GDRIVE upload --delete --chunksize "$CHUNK_SIZE" "$BACKUP_DIR/$f"
    fi
done
{'fraction_non_alphanumeric': 0.2113880742913001, 'fraction_numerical': 0.011241446725317693, 'mean_word_length': 2.2691693290734825, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6394619', 'n_tokens_mistral': 1647, 'n_tokens_neox': 1592, 'n_words': 373}