<commit_before>#!/usr/bin/env bash

set -eu
set -o pipefail

function s3cmd_get_progress() {
	len=0
	while read line; do
		if [[ "$len" -gt 0 ]]; then
			# repeat a backspace $len times
			# need to use seq; {1..$len} doesn't work
			printf '%0.s\b' $(seq 1 $len)
		fi
		echo -n "$line"
		len=${#line}
	done < <(grep --line-buffered -o -E '\[[0-9]+ of [0-9]+\]') # filter only the "[1 of 99]" bits from 's3cmd get' output
}

remove=true

# process flags
optstring=":-:"
while getopts "$optstring" opt; do
	case $opt in
		-)
			case "$OPTARG" in
				no-remove)
					remove=false
					;;
				*)
					echo "Invalid option: --$OPTARG" >&2
					exit 2
					;;
			esac
	esac
done
# clear processed arguments
shift $((OPTIND-1))

if [[ $# -lt "2" || $# -gt "6" ]]; then
	echo "Usage: $(basename $0) [--no-remove] DEST_BUCKET DEST_PREFIX [DEST_REGION [SOURCE_BUCKET SOURCE_PREFIX [SOURCE_REGION]]]" >&2
	echo "  DEST_BUCKET:   destination S3 bucket name." >&2
	echo "  DEST_REGION:   destination bucket region, e.g. us-west-1; default: 's3'." >&2
	echo "  DEST_PREFIX:   destination prefix, e.g. '' or 'dist-stable/'." >&2
	echo "  SOURCE_BUCKET: source S3 bucket name; default: '\$S3_BUCKET'." >&2
	echo "  SOURCE_REGION: source bucket region; default: '\$S3_REGION' or 's3'." >&2
	echo "  SOURCE_PREFIX: source prefix; default: '\${S3_PREFIX}'." >&2
	echo "  --no-remove: no removal of destination packages that are not in source bucket." >&2
	exit 2
fi

dst_bucket=$1; shift
dst_prefix=$1; shift
if [[ $# -gt 2 ]]; then
	# region name given
	dst_region=$1; shift
else
	dst_region="s3"
fi
echo "$@"
src_bucket=${1:-$S3_BUCKET}; shift || true
src_prefix=${1:-$S3_PREFIX}; shift || true
if [[ $# == "1" ]]; then
	# region name given
	src_region=$1; shift
else
	src_region=${S3_REGION:-"s3"}
fi

src_tmp=$(mktemp -d -t "src-repo.XXXXX")
dst_tmp=$(mktemp -d -t "dst-repo.XXXXX")
here=$(cd $(dirname $0); pwd)

# clean up at the end
trap 'rm -rf $src_tmp $dst_tmp;' EXIT

echo -n "Fetching source's manifests from s3://${src_bucket}/${src_prefix}... " >&2
(
	cd $src_tmp
	out=$(s3cmd --ssl get s3://${src_bucket}/${src_prefix}packages.json 2>&1) || { echo -e "No packages.json in source repo:\n$out" >&2; exit 1; }
	s3cmd --ssl --progress get s3://${src_bucket}/${src_prefix}*.composer.json 2>&1 | tee download.log | s3cmd_get_progress >&2 || { echo -e "failed! Error:\n$(cat download.log)" >&2; exit 1; }<commit_msg>fix sync.sh debug-printing additional arguments<commit_after>#!/usr/bin/env bash

set -eu
set -o pipefail

function s3cmd_get_progress() {
	len=0
	while read line; do
		if [[ "$len" -gt 0 ]]; then
			# repeat a backspace $len times
			# need to use seq; {1..$len} doesn't work
			printf '%0.s\b' $(seq 1 $len)
		fi
		echo -n "$line"
		len=${#line}
	done < <(grep --line-buffered -o -E '\[[0-9]+ of [0-9]+\]') # filter only the "[1 of 99]" bits from 's3cmd get' output
}

remove=true

# process flags
optstring=":-:"
while getopts "$optstring" opt; do
	case $opt in
		-)
			case "$OPTARG" in
				no-remove)
					remove=false
					;;
				*)
					echo "Invalid option: --$OPTARG" >&2
					exit 2
					;;
			esac
	esac
done
# clear processed arguments
shift $((OPTIND-1))

if [[ $# -lt "2" || $# -gt "6" ]]; then
	echo "Usage: $(basename $0) [--no-remove] DEST_BUCKET DEST_PREFIX [DEST_REGION [SOURCE_BUCKET SOURCE_PREFIX [SOURCE_REGION]]]" >&2
	echo "  DEST_BUCKET:   destination S3 bucket name." >&2
	echo "  DEST_REGION:   destination bucket region, e.g. us-west-1; default: 's3'." >&2
	echo "  DEST_PREFIX:   destination prefix, e.g. '' or 'dist-stable/'." >&2
	echo "  SOURCE_BUCKET: source S3 bucket name; default: '\$S3_BUCKET'." >&2
	echo "  SOURCE_REGION: source bucket region; default: '\$S3_REGION' or 's3'." >&2
	echo "  SOURCE_PREFIX: source prefix; default: '\${S3_PREFIX}'." >&2
	echo "  --no-remove: no removal of destination packages that are not in source bucket." >&2
	exit 2
fi

dst_bucket=$1; shift
dst_prefix=$1; shift
if [[ $# -gt 2 ]]; then
	# region name given
	dst_region=$1; shift
else
	dst_region="s3"
fi

src_bucket=${1:-$S3_BUCKET}; shift || true
src_prefix=${1:-$S3_PREFIX}; shift || true
if [[ $# == "1" ]]; then
	# region name given
	src_region=$1; shift
else
	src_region=${S3_REGION:-"s3"}
fi

src_tmp=$(mktemp -d -t "src-repo.XXXXX")
dst_tmp=$(mktemp -d -t "dst-repo.XXXXX")
here=$(cd $(dirname $0); pwd)

# clean up at the end
trap 'rm -rf $src_tmp $dst_tmp;' EXIT

echo -n "Fetching source's manifests from s3://${src_bucket}/${src_prefix}... " >&2
(
	cd $src_tmp
	out=$(s3cmd --ssl get s3://${src_bucket}/${src_prefix}packages.json 2>&1) || { echo -e "No packages.json in source repo:\n$out" >&2; exit 1; }
	s3cmd --ssl --progress get s3://${src_bucket}/${src_prefix}*.composer.json 2>&1 | tee download.log | s3cmd_get_progress >&2 || { echo -e "failed! Error:\n$(cat download.log)" >&2; exit 1; }
{'fraction_non_alphanumeric': 0.18482530494107918, 'fraction_numerical': 0.027703121769691957, 'mean_word_length': 3.5901328273244784, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6316008', 'n_tokens_mistral': 2022, 'n_tokens_neox': 1908, 'n_words': 559}