<commit_before>#!/bin/bash
# Publishes built DEBs to an s3-backed DEB repo.
if [ ! -z "${DEBUG}" ]; then
  set -x
fi

SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
SRC_BASE="${SCRIPT_DIR}/../.."

DEPENDENCIES=("aws" "reprepro")
REGION="us-east-1"
SOURCE_DIR=""
TARGET_BUCKET=""

for dep in "${DEPENDENCIES[@]}"
do
  if [ ! $(which ${dep}) ]; then
      echo "${dep} must be available."
      exit 1
  fi
done

while getopts "s:t:" opt; do
  case $opt in
    s) SOURCE_DIR=$OPTARG ;;
    t) TARGET_BUCKET=$OPTARG ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      exit 1
      ;;
  esac
done

if [ -z "${SOURCE_DIR}" ]; then
  echo "Source directory must be specified."
  exit 1
fi

if [ -z "${TARGET_BUCKET}" ]; then
  echo "Target bucket must be specified."
  exit 1
fi

# check if repo alredy exists in S3
GPG_KEY="<KEY>"
wget https://s3.amazonaws.com/${TARGET_BUCKET}/dists/xenial/InRelease
set -e
if [ $? -eq 0 ]; then
  # mirror and append
  # existing repo
  # mirror repot from S3
  wget -O - ${TARGET_BUCKET}/public.key | gpg --no-default-keyring --keyring trustedkeys.gpg --import
  aptly mirror create ${TARGET_BUCKET} https://s3.amazonaws.com/${TARGET_BUCKET}/ xenial
  aptly mirror update ${TARGET_BUCKET}
  # create local repo
  aptly repo create -distribution=xenial ${TARGET_BUCKET}
  # import existing packages
  aptly repo import ${TARGET_BUCKET} ${TARGET_BUCKET} kismatic-docker-engine kismatic-etcd kismatic-kubernetes-master kismatic-kubernetes-networking kismatic-kubernetes-node
  # add new packages
  aptly repo add ${TARGET_BUCKET} $SOURCE_DIR
  # push to S3
  aptly publish repo -gpg-key=${GPG_KEY} ${TARGET_BUCKET} s3:${TARGET_BUCKET}:
else
  # new repo
  # create local repo
  aptly repo create -distribution=xenial ${TARGET_BUCKET}
  aptly repo add ${TARGET_BUCKET} $SOURCE_DIR
  # push to S3
  aptly publish repo -gpg-key=${GPG_KEY} ${TARGET_BUCKET} s3:${TARGET_BUCKET}:
fi
<commit_msg>Dont exit on wget test again again<commit_after>#!/bin/bash
# Publishes built DEBs to an s3-backed DEB repo.
if [ ! -z "${DEBUG}" ]; then
  set -x
fi

SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
SRC_BASE="${SCRIPT_DIR}/../.."

DEPENDENCIES=("aws" "reprepro")
REGION="us-east-1"
SOURCE_DIR=""
TARGET_BUCKET=""

for dep in "${DEPENDENCIES[@]}"
do
  if [ ! $(which ${dep}) ]; then
      echo "${dep} must be available."
      exit 1
  fi
done

while getopts "s:t:" opt; do
  case $opt in
    s) SOURCE_DIR=$OPTARG ;;
    t) TARGET_BUCKET=$OPTARG ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      exit 1
      ;;
  esac
done

if [ -z "${SOURCE_DIR}" ]; then
  echo "Source directory must be specified."
  exit 1
fi

if [ -z "${TARGET_BUCKET}" ]; then
  echo "Target bucket must be specified."
  exit 1
fi

# check if repo alredy exists in S3
GPG_KEY="<KEY>"
wget https://s3.amazonaws.com/${TARGET_BUCKET}/dists/xenial/InRelease
if [ $? -eq 0 ]; then
  set -e
  # mirror and append
  # existing repo
  # mirror repot from S3
  wget -O - ${TARGET_BUCKET}/public.key | gpg --no-default-keyring --keyring trustedkeys.gpg --import
  aptly mirror create ${TARGET_BUCKET} https://s3.amazonaws.com/${TARGET_BUCKET}/ xenial
  aptly mirror update ${TARGET_BUCKET}
  # create local repo
  aptly repo create -distribution=xenial ${TARGET_BUCKET}
  # import existing packages
  aptly repo import ${TARGET_BUCKET} ${TARGET_BUCKET} kismatic-docker-engine kismatic-etcd kismatic-kubernetes-master kismatic-kubernetes-networking kismatic-kubernetes-node
  # add new packages
  aptly repo add ${TARGET_BUCKET} $SOURCE_DIR
  # push to S3
  aptly publish repo -gpg-key=${GPG_KEY} ${TARGET_BUCKET} s3:${TARGET_BUCKET}:
else
  set -e
  # new repo
  # create local repo
  aptly repo create -distribution=xenial ${TARGET_BUCKET}
  aptly repo add ${TARGET_BUCKET} $SOURCE_DIR
  # push to S3
  aptly publish repo -gpg-key=${GPG_KEY} ${TARGET_BUCKET} s3:${TARGET_BUCKET}:
fi

{'fraction_non_alphanumeric': 0.14048106448311157, 'fraction_numerical': 0.008700102354145343, 'mean_word_length': 3.732445520581114, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5734156', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1438, 'n_words': 406}