<commit_before>#!/bin/bash
set -eou pipefail

source "$HOME/.duplicity/config"

export PASSPHRASE
export AWS_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY
export S3_BUCKET_NAME
export LOCAL_DIRECTORY_TO_BACK_UP

RELATIVE_PATH_TO_VERIFY=$1

duplicity verify \
    --name trw-personal \
    --verbosity Notice \
    --compare-data \
    --file-to-restore $RELATIVE_PATH_TO_VERIFY \
    s3://s3.amazonaws.com/$S3_BUCKET_NAME $LOCAL_DIRECTORY_TO_BACK_UP/$RELATIVE_PATH_TO_VERIFY

unset PASSPHRASE
unset AWS_ACCESS_KEY_ID
unset AWS_SECRET_ACCESS_KEY
unset S3_BUCKET_NAME
unset LOCAL_DIRECTORY_TO_BACK_UP
<commit_msg>Note how to verify a random sample of files<commit_after>#!/bin/bash
set -eou pipefail

# You can use this command to find 5 random files which you can then run this script individually on:
# find /path/to/your/local/directory/to/back_up -type f | shuf -n5
# TODO: write a script which wraps finding these 5 random files and verifies them

source "$HOME/.duplicity/config"

export PASSPHRASE
export AWS_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY
export S3_BUCKET_NAME
export LOCAL_DIRECTORY_TO_BACK_UP

RELATIVE_PATH_TO_VERIFY=$1

duplicity verify \
    --name trw-personal \
    --verbosity Notice \
    --compare-data \
    --file-to-restore "$RELATIVE_PATH_TO_VERIFY" \
    s3://s3.amazonaws.com/$S3_BUCKET_NAME "$LOCAL_DIRECTORY_TO_BACK_UP/$RELATIVE_PATH_TO_VERIFY"

unset PASSPHRASE
unset AWS_ACCESS_KEY_ID
unset AWS_SECRET_ACCESS_KEY
unset S3_BUCKET_NAME
unset LOCAL_DIRECTORY_TO_BACK_UP

{'fraction_non_alphanumeric': 0.07123834886817576, 'fraction_numerical': 0.009986684420772303, 'mean_word_length': 4.87109375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6042884', 'n_tokens_mistral': 569, 'n_tokens_neox': 565, 'n_words': 123}