<commit_before>#!/bin/bash
#
# Creates LLVM SVN snapshots: llvm-$REV.tar.bz2 and llvm-gcc-4.2-$REV.tar.bz2,
# where $REV is an SVN revision of LLVM.  This is used for creating stable
# tarballs which can be used to build known-to-work crosstools.
#
# Syntax:
#   $0 [REV] -- grabs the revision $REV from SVN; if not specified, grabs the
#   latest SVN revision.

set -o nounset
set -o errexit

readonly REV="${1:-HEAD}"

runOnModule() {
  local module=$1
  local log="${module}.log"
  echo "Running: svn co -r ${REV} ${module}; log in ${log}"
  svn co -r ${REV} http://llvm.org/svn/llvm-project/${module}/trunk ${module} \
      > ${log} 2>&1

  # Delete all the ".svn" dirs; they take quite a lot of space.
  echo "Cleaning up .svn dirs"
  find ${module} -type d -name \.svn -print0 | xargs -0 /bin/rm -rf

  # Create "module-revision.tar.bz2" packages from the SVN checkout dirs.
  local revision=$(grep "Checked out revision" ${log} | \
                   sed 's/[^0-9]\+\([0-9]\+\)[^0-9]\+/\1/')
  local tarball="${module}-${revision}.tar.bz2"
  echo "Creating tarball: ${tarball}"
  tar cjf ${tarball} ${module}

  echo "Cleaning SVN checkout dir ${module}"
  rm -rf ${module} ${log}
}

for module in "llvm" "llvm-gcc-4.2"; do
  runOnModule ${module}
done
<commit_msg>* Use "svn export" instead of "svn co" and avoid cleaning up .svn dirs * Use "svn info" to get last revision in repo, will get matching tarballs * Now run "svn -q" since "svn info" tells us the revision number<commit_after>#!/bin/bash
#
# Creates LLVM SVN snapshots: llvm-$REV.tar.bz2 and llvm-gcc-4.2-$REV.tar.bz2,
# where $REV is an SVN revision of LLVM.  This is used for creating stable
# tarballs which can be used to build known-to-work crosstools.
#
# Syntax:
#   $0 [REV] -- grabs the revision $REV from SVN; if not specified, grabs the
#   latest SVN revision.

set -o nounset
set -o errexit

readonly LLVM_PROJECT_SVN="http://llvm.org/svn/llvm-project"

getLatestRevisionFromSVN() {
  svn info ${LLVM_PROJECT_SVN} | egrep ^Revision | sed 's/^Revision: //'
}

readonly REV="${1:-$(getLatestRevisionFromSVN)}"

createTarballFromSVN() {
  local module=$1
  local log="${module}.log"
  echo "Running: svn export -r ${REV} ${module}; log in ${log}"
  svn -q export -r ${REV} ${LLVM_PROJECT_SVN}/${module}/trunk \
      ${module} > ${log} 2>&1

  # Create "module-revision.tar.bz2" packages from the SVN checkout dirs.
  local tarball="${module}-${REV}.tar.bz2"
  echo "Creating tarball: ${tarball}"
  tar cjf ${tarball} ${module}

  echo "Cleaning up '${module}'"
  rm -rf ${module} ${log}
}

for module in "llvm" "llvm-gcc-4.2"; do
  createTarballFromSVN ${module}
done

{'fraction_non_alphanumeric': 0.16043790109475273, 'fraction_numerical': 0.01321253303133258, 'mean_word_length': 3.537671232876712, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5911321', 'n_tokens_mistral': 1028, 'n_tokens_neox': 986, 'n_words': 315}