<commit_before>#!/bin/bash
set -euo pipefail

artifacts_build=$(buildkite-agent meta-data get "agent-artifacts-build")

dry_run() {
  if [[ "${DRY_RUN:-}" == "false" ]] ; then
    "$@"
  else
    echo "[dry-run] $*"
  fi
}

if [[ "$CODENAME" == "" ]]; then
  echo "Error: Missing \$CODENAME (stable or unstable)"
  exit 1
fi

YUM_PATH=/yum.buildkite.com

echo '--- Downloading built yum packages packages'
rm -rf rpm
mkdir -p rpm
buildkite-agent artifact download --build "$artifacts_build" "rpm/*.rpm" rpm/

echo '--- Installing dependencies'
bundle

# Make sure we have a local copy of the yum repo
echo "--- Syncing s3://$RPM_S3_BUCKET to `hostname`"
mkdir -p $YUM_PATH
dry_run aws --region us-east-1 s3 sync --delete "s3://$RPM_S3_BUCKET" "$YUM_PATH"

# Add the rpms and update meta-data
for ARCH in "x86_64" "i386"; do
  echo "--- Updating yum repository for ${CODENAME}/${ARCH}"

  ARCH_PATH="${YUM_PATH}/buildkite-agent/${CODENAME}/${ARCH}"
  mkdir -p "$ARCH_PATH"
  find "rpm/" -type f -name "*${ARCH}*" | xargs cp -t "$ARCH_PATH"
  # createrepo_c is much faster and more resilient than createrepo
  dry_run createrepo_c --no-database --unique-md-filenames --retain-old-md-by-age=7d --update "$ARCH_PATH" || \
    dry_run createrepo_c --no-database --unique-md-filenames --retain-old-md-by-age=7d "$ARCH_PATH"
  #createrepo --no-database --unique-md-filenames --update "$ARCH_PATH" || \
  #  createrepo --no-database --unique-md-filenames "$ARCH_PATH"
done

# Sync back our changes to S3<commit_msg>Dry run can download and index, but not upload<commit_after>#!/bin/bash
set -euo pipefail

artifacts_build=$(buildkite-agent meta-data get "agent-artifacts-build")

dry_run() {
  if [[ "${DRY_RUN:-}" == "false" ]] ; then
    "$@"
  else
    echo "[dry-run] $*"
  fi
}

if [[ "$CODENAME" == "" ]]; then
  echo "Error: Missing \$CODENAME (stable or unstable)"
  exit 1
fi

YUM_PATH=/yum.buildkite.com

echo '--- Downloading built yum packages packages'
rm -rf rpm
mkdir -p rpm
buildkite-agent artifact download --build "$artifacts_build" "rpm/*.rpm" rpm/

echo '--- Installing dependencies'
bundle

# Make sure we have a local copy of the yum repo
echo "--- Syncing s3://$RPM_S3_BUCKET to `hostname`"
mkdir -p $YUM_PATH
aws --region us-east-1 s3 sync --delete "s3://$RPM_S3_BUCKET" "$YUM_PATH"

# Add the rpms and update meta-data
for ARCH in "x86_64" "i386"; do
  echo "--- Updating yum repository for ${CODENAME}/${ARCH}"

  ARCH_PATH="${YUM_PATH}/buildkite-agent/${CODENAME}/${ARCH}"
  mkdir -p "$ARCH_PATH"
  find "rpm/" -type f -name "*${ARCH}*" | xargs cp -t "$ARCH_PATH"
  # createrepo_c is much faster and more resilient than createrepo
  createrepo_c --no-database --unique-md-filenames --retain-old-md-by-age=7d --update "$ARCH_PATH" || \
    createrepo_c --no-database --unique-md-filenames --retain-old-md-by-age=7d "$ARCH_PATH"
  #createrepo --no-database --unique-md-filenames --update "$ARCH_PATH" || \
  #  createrepo --no-database --unique-md-filenames "$ARCH_PATH"
done

# Sync back our changes to S3
{'fraction_non_alphanumeric': 0.1650678134303672, 'fraction_numerical': 0.011247105524313596, 'mean_word_length': 3.581818181818182, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1185195', 'n_tokens_mistral': 1193, 'n_tokens_neox': 1116, 'n_words': 308}