<commit_before>#!/usr/bin/env bash

FILES_DESTINATION="$CURRENT_SUBVOL/files"

if [[ ${BACKUP_FILES_PATHS+defined} = defined ]]; then
    if [ ! -d "$FILES_DESTINATION" ]; then
        mkdir -p "$FILES_DESTINATION"
    fi

    includes=""

    orig_IFS=$IFS
    IFS=:
    for path in $BACKUP_FILES_PATHS; do
        if [[ "$path" = /* ]]; then
            if [ -d "$path" ]; then
                includes="--include='${path/%\//}/***' $includes"

                subpath="${path/%\//}"

                while [ -d "$subpath" ]; do
                    includes="--include='${subpath}/' $includes"
                    subpath="${subpath%/*}"
                done

            else
                echo "[Backup - Files] Given path '$path' is not known" >&2
                exit 1
            fi
        else
            echo "[Backup - Files] Path '$path' is not absolute" >&2
            exit 1
        fi
    done
    IFS=$orig_IFS

    eval "rsync --archive --numeric-ids --inplace --relative --delete --delete-excluded $includes --exclude='*' / $FILES_DESTINATION"
else
    rm -rf "$FILES_DESTINATION"
fi<commit_msg>Changed erroneous IFS trickery to splitting using tr<commit_after>#!/usr/bin/env bash

FILES_DESTINATION="$CURRENT_SUBVOL/files"

if [[ ${BACKUP_FILES_PATHS+defined} = defined ]]; then
    if [ ! -d "$FILES_DESTINATION" ]; then
        mkdir -p "$FILES_DESTINATION"
    fi

    includes=""

    paths=($(echo $BACKUP_FILES_PATHS | tr ":" "\n"))
    
    for path in $paths; do
        if [[ "$path" = /* ]]; then
            if [ -d "$path" ]; then
                includes="--include='${path/%\//}/***' $includes"

                subpath="${path/%\//}"

                while [ -d "$subpath" ]; do
                    includes="--include='${subpath}/' $includes"
                    subpath="${subpath%/*}"
                done

            else
                echo "[Backup - Files] Given path '$path' is not known" >&2
                exit 1
            fi
        else
            echo "[Backup - Files] Path '$path' is not absolute" >&2
            exit 1
        fi
    done

    eval "rsync --archive --numeric-ids --inplace --relative --delete --delete-excluded $includes --exclude='*' / $FILES_DESTINATION"
else
    rm -rf "$FILES_DESTINATION"
fi
{'fraction_non_alphanumeric': 0.1564835164835165, 'fraction_numerical': 0.0035164835164835165, 'mean_word_length': 1.668229777256741, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1020858', 'n_tokens_mistral': 738, 'n_tokens_neox': 687, 'n_words': 179}