<commit_before>#!/bin/bash
set -e

PREFIX="spack:"

echo "$PREFIX Beginning setup..."
if [[ $EUID -ne 0 ]]; then
  echo "$PREFIX This script must be run as root"
  exit 1
fi

# Only install and configure spack if ${INSTALL_DIR} doesn't exist
if [ ! -d ${INSTALL_DIR} ]; then

  # Install spack
  echo "$PREFIX Installing spack from ${SPACK_URL}..."
  {
  mkdir -p ${INSTALL_DIR};
  chmod a+rwx ${INSTALL_DIR};
  chmod a+s ${INSTALL_DIR};
  cd ${INSTALL_DIR};
  git clone ${SPACK_URL} .
  } &>> ${LOG_FILE}
  echo "$PREFIX Checking out ${SPACK_REF}..."
  git checkout ${SPACK_REF} >> ${LOG_FILE} 2>&1

  {
  source ${INSTALL_DIR}/share/spack/setup-env.sh;
  spack compiler find --scope site
  } &>> ${LOG_FILE} 2>&1

  echo "$PREFIX Configuring spack..."
  %{for c in CONFIGS ~}
    %{if c.type == "single-config" ~}
      spack config --scope=${c.scope} add "${c.value}" >> ${LOG_FILE} 2>&1
    %{endif ~}

    %{if c.type == "file" ~}
      {
      cat <<EOF > ${INSTALL_DIR}/spack_conf.yaml
${c.value}
EOF

      spack config --scope=${c.scope} add -f ${INSTALL_DIR}/spack_conf.yaml
      rm -f ${INSTALL_DIR}/spack_conf.yaml
      } &>> ${LOG_FILE} 2>&1
    %{endif ~}
  %{endfor ~}

  echo "$PREFIX Setting up spack mirrors..."
  %{for m in MIRRORS ~}
  spack mirror add --scope site ${m.mirror_name} ${m.mirror_url} >> ${LOG_FILE} 2>&1
  %{endfor ~}

  echo "$PREFIX Installing GPG keys"
  spack gpg init >> ${LOG_FILE} 2>&1
  %{for k in GPG_KEYS ~}
    %{if k.type == "file" ~}
      spack gpg trust ${k.path}
    %{endif ~}

    %{if k.type == "new" ~}
      spack gpg create "${k.name}" ${k.email}
    %{endif ~}
  %{endfor ~}

  spack buildcache keys --install --trust >> ${LOG_FILE} 2>&1
else
  source ${INSTALL_DIR}/share/spack/setup-env.sh >> ${LOG_FILE} 2>&1
fi

echo "$PREFIX Installing licenses..."
%{for lic in LICENSES ~}
  gsutil cp ${lic.source} ${lic.dest} >> ${LOG_FILE} 2>&1
%{endfor ~}

echo "$PREFIX Installing compilers..."
%{for c in COMPILERS ~}
  {
    spack install ${c};
    spack load ${c};
    spack clean -s
  } &>> ${LOG_FILE}
%{endfor ~}

spack compiler find --scope site >> ${LOG_FILE} 2>&1

echo "$PREFIX Installing root spack specs..."
%{for p in PACKAGES ~}
  spack install ${p} >> ${LOG_FILE} 2>&1
  spack clean -s
%{endfor ~}

echo "$PREFIX Configuring spack environments"
%{for e in ENVIRONMENTS ~}
  {
    spack env create ${e.name};
    spack env activate ${e.name};
  } &>> ${LOG_FILE}

  echo "$PREFIX    Configuring spack environment ${e.name}"
  %{for p in e.packages ~}
    spack add ${p} >> ${LOG_FILE} 2>&1
  %{endfor ~}

  echo "$PREFIX    Concretizing spack environment ${e.name}"
  spack concretize >> ${LOG_FILE} 2>&1
  echo "$PREFIX    Installing packages for spack environment ${e.name}"
  spack install >> ${LOG_FILE} 2>&1

  spack env deactivate >> ${LOG_FILE} 2>&1
  spack clean -s
%{endfor ~}

echo "$PREFIX Populating defined buildcaches"
%{for c in CACHES_TO_POPULATE ~}
  %{if c.type == "directory" ~}
    # shellcheck disable=SC2046  
    spack buildcache create -d ${c.path} -af $(spack find --format /{hash})
    spack gpg publish --rebuild-index -d ${c.path}
  %{endif ~}
  %{if c.type == "mirror" ~} 
    # shellcheck disable=SC2046  
    spack buildcache create --mirror-url ${c.path} -af $(spack find --format /{hash})
    spack gpg publish --rebuild-index --mirror-url ${c.path}
  %{endif ~}
%{endfor ~}

echo "source /apps/spack/share/spack/setup-env.sh" >> /etc/profile.d/spack.sh
chmod a+rx /etc/profile.d/spack.sh<commit_msg>Add index rebuilding to populated caches<commit_after>#!/bin/bash
set -e

PREFIX="spack:"

echo "$PREFIX Beginning setup..."
if [[ $EUID -ne 0 ]]; then
  echo "$PREFIX This script must be run as root"
  exit 1
fi

# Only install and configure spack if ${INSTALL_DIR} doesn't exist
if [ ! -d ${INSTALL_DIR} ]; then

  # Install spack
  echo "$PREFIX Installing spack from ${SPACK_URL}..."
  {
  mkdir -p ${INSTALL_DIR};
  chmod a+rwx ${INSTALL_DIR};
  chmod a+s ${INSTALL_DIR};
  cd ${INSTALL_DIR};
  git clone ${SPACK_URL} .
  } &>> ${LOG_FILE}
  echo "$PREFIX Checking out ${SPACK_REF}..."
  git checkout ${SPACK_REF} >> ${LOG_FILE} 2>&1

  {
  source ${INSTALL_DIR}/share/spack/setup-env.sh;
  spack compiler find --scope site
  } &>> ${LOG_FILE} 2>&1

  echo "$PREFIX Configuring spack..."
  %{for c in CONFIGS ~}
    %{if c.type == "single-config" ~}
      spack config --scope=${c.scope} add "${c.value}" >> ${LOG_FILE} 2>&1
    %{endif ~}

    %{if c.type == "file" ~}
      {
      cat <<EOF > ${INSTALL_DIR}/spack_conf.yaml
${c.value}
EOF

      spack config --scope=${c.scope} add -f ${INSTALL_DIR}/spack_conf.yaml
      rm -f ${INSTALL_DIR}/spack_conf.yaml
      } &>> ${LOG_FILE} 2>&1
    %{endif ~}
  %{endfor ~}

  echo "$PREFIX Setting up spack mirrors..."
  %{for m in MIRRORS ~}
  spack mirror add --scope site ${m.mirror_name} ${m.mirror_url} >> ${LOG_FILE} 2>&1
  %{endfor ~}

  echo "$PREFIX Installing GPG keys"
  spack gpg init >> ${LOG_FILE} 2>&1
  %{for k in GPG_KEYS ~}
    %{if k.type == "file" ~}
      spack gpg trust ${k.path}
    %{endif ~}

    %{if k.type == "new" ~}
      spack gpg create "${k.name}" ${k.email}
    %{endif ~}
  %{endfor ~}

  spack buildcache keys --install --trust >> ${LOG_FILE} 2>&1
else
  source ${INSTALL_DIR}/share/spack/setup-env.sh >> ${LOG_FILE} 2>&1
fi

echo "$PREFIX Installing licenses..."
%{for lic in LICENSES ~}
  gsutil cp ${lic.source} ${lic.dest} >> ${LOG_FILE} 2>&1
%{endfor ~}

echo "$PREFIX Installing compilers..."
%{for c in COMPILERS ~}
  {
    spack install ${c};
    spack load ${c};
    spack clean -s
  } &>> ${LOG_FILE}
%{endfor ~}

spack compiler find --scope site >> ${LOG_FILE} 2>&1

echo "$PREFIX Installing root spack specs..."
%{for p in PACKAGES ~}
  spack install ${p} >> ${LOG_FILE} 2>&1
  spack clean -s
%{endfor ~}

echo "$PREFIX Configuring spack environments"
%{for e in ENVIRONMENTS ~}
  {
    spack env create ${e.name};
    spack env activate ${e.name};
  } &>> ${LOG_FILE}

  echo "$PREFIX    Configuring spack environment ${e.name}"
  %{for p in e.packages ~}
    spack add ${p} >> ${LOG_FILE} 2>&1
  %{endfor ~}

  echo "$PREFIX    Concretizing spack environment ${e.name}"
  spack concretize >> ${LOG_FILE} 2>&1
  echo "$PREFIX    Installing packages for spack environment ${e.name}"
  spack install >> ${LOG_FILE} 2>&1

  spack env deactivate >> ${LOG_FILE} 2>&1
  spack clean -s
%{endfor ~}

echo "$PREFIX Populating defined buildcaches"
%{for c in CACHES_TO_POPULATE ~}
  %{if c.type == "directory" ~}
    # shellcheck disable=SC2046
    spack buildcache create -d ${c.path} -af $(spack find --format /{hash})
    spack gpg publish -d ${c.path}
    spack buildcache update-index -d ${c.path} --keys
  %{endif ~}
  %{if c.type == "mirror" ~}
    # shellcheck disable=SC2046
    spack buildcache create --mirror-url ${c.path} -af $(spack find --format /{hash})
    spack gpg publish --mirror-url ${c.path}
    spack buildcache update-index -mirror-url ${c.path} --keys
  %{endif ~}
%{endfor ~}

echo "source /apps/spack/share/spack/setup-env.sh" >> /etc/profile.d/spack.sh
chmod a+rx /etc/profile.d/spack.sh
{'fraction_non_alphanumeric': 0.1801941201294134, 'fraction_numerical': 0.011253340835560557, 'mean_word_length': 3.1578947368421053, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6077425', 'n_tokens_mistral': 2797, 'n_tokens_neox': 2685, 'n_words': 756}