<commit_before>#!/bin/sh

# streamsync [OPTIONS] FILE LOCATION
# calls 'rsync --inplace OPTIONS FILE LOCATION' repeatedly until FILE stops being modified

#TODO: replace .* with a more specific list of extensions

# handling temporary file extensions should be optional, not assumed

OPTIONS="$1"
FILE="$2"
LOCATION="$3"
FILE_INFO=""
TIMEOUT="${4-10}" # if the third argument is empty, default to 10

# if $2 does not exist, look for $2.* (chrome uses .crdownload, firefox uses .part, etc.) 
# this needs to be more specific
if [! -e "$FILE" ]; then
	FILE="$2".*
fi

# what happens when a temporary file is deleted?
while [ "$FILE_INFO" = `ls -l $FILE` ]; do
	FILE_INFO=`ls -l $FILE`
	rsync --inplace "$OPTIONS" "$FILE" "$LOCATION"<commit_msg>significant changes, works but not finished<commit_after>#!/bin/sh

set -e

# Description:
#   Creates a link to the file and then repeatedly calls rsync --inplace until
#   the link stops existing. This means that the only way to stop the script is
#   to either kill the process or delete the autogenerated hard link that it
#   depends on.

USAGE='streamsync [-t TIMEOUT] [-o OPTIONS] FILE DEST'
if [ $# -lt 2 -o $# -gt 6 ]; then
	echo $USAGE && exit 1
fi

TIMEOUT=10
OPTIONS=""

while getopts ho:t: ARG; do
	case $ARG in
	h) echo $USAGE && exit 0 ;;
	o) OPTIONS=$OPTARG ;;
	t) TIMEOUT=$OPTARG ;;
	?) echo $USAGE; exit 1 ;;
	esac
done

shift $(( $OPTIND - 1 ))

FILE="$1"
DEST="$2"

host="$(echo "$DEST" | cut -d ':' -f 1)"

if [ '!' "$host" = "$DEST" ]; then
	# We are copying to a remote location
	ssh -MNf -O "exit" "$host"
fi

STREAM="$FILE.stream"
# TODO: use temporary files instead of this
# TODO: create an easy way to stop the sync
ln "$FILE" "$STREAM"
# TODO: don't want to defaultly name the destination file $FILE.stream
while [ -e "$STREAM" ]; do
	rsync --inplace $OPTIONS "$STREAM" "$DEST"
{'fraction_non_alphanumeric': 0.14061654948620878, 'fraction_numerical': 0.011898323418063819, 'mean_word_length': 3.8684210526315788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2302374', 'n_tokens_mistral': 651, 'n_tokens_neox': 623, 'n_words': 260}