<commit_before>#!/bin/bash
#
# Compiles and installs a Linux/x86_64 -> Linux/ARM crosstool based on LLVM and
# LLVM-GCC-4.2 using SVN snapshots in provided tarballs.

set -o nounset
set -o errexit

echo -n "Welcome to LLVM Linux/X86_64 -> Linux/ARM crosstool "
echo "builder/installer; some steps will require sudo privileges."

readonly INSTALL_ROOT="${INSTALL_ROOT:-/usr/local}"
# Both $USER and root *must* have read/write access to this dir.
readonly SCRATCH_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/llvm-project.XXXXXX")
readonly SRC_ROOT="${SCRATCH_ROOT}/src"
readonly OBJ_ROOT="${SCRATCH_ROOT}/obj"

readonly CROSS_HOST="x86_64-unknown-linux-gnu"
readonly CROSS_TARGET="arm-none-linux-gnueabi"
readonly CROSS_MARCH="${CROSS_MARCH:-armv6}"

readonly CODE_SOURCERY="${INSTALL_ROOT}/codesourcery"
readonly CODE_SOURCERY_PKG_PATH="${CODE_SOURCERY_PKG_PATH:-${HOME}/codesourcery}"
readonly CODE_SOURCERY_HTTP="http://www.codesourcery.com/sgpp/lite/arm/portal/package1787/public"
readonly CODE_SOURCERY_PKG="arm-2007q3-51-arm-none-linux-gnueabi-i686-pc-linux-gnu.tar.bz2"
readonly CODE_SOURCERY_ROOT="${CODE_SOURCERY}/arm-2007q3"
readonly CODE_SOURCERY_BIN="${CODE_SOURCERY_ROOT}/bin"
# Make sure ${CROSS_TARGET}-* binutils are in command path
export PATH="${CODE_SOURCERY_BIN}:${PATH}"

readonly CROSS_TARGET_AS="${CODE_SOURCERY_BIN}/${CROSS_TARGET}-as"
readonly CROSS_TARGET_LD="${CODE_SOURCERY_BIN}/${CROSS_TARGET}-ld"

readonly SYSROOT="${CODE_SOURCERY_ROOT}/${CROSS_TARGET}/libc"

readonly LLVM_PROJECT="${INSTALL_ROOT}/llvm-project"
readonly LLVM_INSTALL_ROOT="${LLVM_PROJECT}/${CROSS_HOST}/${CROSS_TARGET}"
readonly LLVM_PKG_PATH="${LLVM_PKG_PATH:-${HOME}/llvm-project/snapshots}"

# Latest SVN revision known to be working in this configuration.
readonly LLVM_DEFAULT_REV="70786"

readonly LLVM_PKG="llvm-${LLVM_SVN_REV:-${LLVM_DEFAULT_REV}}.tar.bz2"
readonly LLVM_SRC_DIR="${SRC_ROOT}/llvm"
readonly LLVM_OBJ_DIR="${OBJ_ROOT}/llvm"
readonly LLVM_INSTALL_DIR="${LLVM_INSTALL_ROOT}/llvm"

readonly LLVMGCC_PKG="llvm-gcc-4.2-${LLVMGCC_SVN_REV:-${LLVM_DEFAULT_REV}}.tar.bz2"
readonly LLVMGCC_SRC_DIR="${SRC_ROOT}/llvm-gcc-4.2"
readonly LLVMGCC_OBJ_DIR="${OBJ_ROOT}/llvm-gcc-4.2"
readonly LLVMGCC_INSTALL_DIR="${LLVM_INSTALL_ROOT}/llvm-gcc-4.2"

readonly MAKE_OPTS="-j2"

# Verify we aren't going to install into an existing directory as this might
# create problems as we won't have a clean install.
verifyNotDir() {
  if [[ -d $1 ]]; then
    echo "Install dir $1 already exists; remove it to continue."
    exit
  fi
}

# Params:
#   $1: directory to be created
#   $2: optional mkdir command prefix, e.g. "sudo"
createDir() {
  if [[ ! -e $1 ]]; then
    ${2:-} mkdir -p $1
  elif [[ -e $1 && ! -d $1 ]]; then
    echo "$1 exists but is not a directory; exiting."
    exit 3
  fi
}

sudoCreateDir() {
  createDir $1 sudo
  sudo chown ${USER} $1
}

# Prints out and runs the command, but without logging -- intended for use with
# lightweight commands that don't have useful output to parse, e.g. mkdir, tar,
# etc.<commit_msg>* Install crosstool in /usr/local/crosstool * Updated LLVM and LLVM-GCC to newer revisions * Allow separate revisions per SVN module * Allow user to override MAKE_OPTS * Removed unused function verifyNotDir()<commit_after>#!/bin/bash
#
# Compiles and installs a Linux/x86_64 -> Linux/ARM crosstool based on LLVM and
# LLVM-GCC-4.2 using SVN snapshots in provided tarballs.

set -o nounset
set -o errexit

echo -n "Welcome to LLVM Linux/X86_64 -> Linux/ARM crosstool "
echo "builder/installer; some steps will require sudo privileges."

readonly INSTALL_ROOT="${INSTALL_ROOT:-/usr/local/crosstool}"
# Both $USER and root *must* have read/write access to this dir.
readonly SCRATCH_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/llvm-project.XXXXXX")
readonly SRC_ROOT="${SCRATCH_ROOT}/src"
readonly OBJ_ROOT="${SCRATCH_ROOT}/obj"

readonly CROSS_HOST="x86_64-unknown-linux-gnu"
readonly CROSS_TARGET="arm-none-linux-gnueabi"
readonly CROSS_MARCH="${CROSS_MARCH:-armv6}"

readonly CODE_SOURCERY="${INSTALL_ROOT}/codesourcery"
readonly CODE_SOURCERY_PKG_PATH="${CODE_SOURCERY_PKG_PATH:-${HOME}/codesourcery}"
readonly CODE_SOURCERY_HTTP="http://www.codesourcery.com/sgpp/lite/arm/portal/package1787/public"
readonly CODE_SOURCERY_PKG="arm-2007q3-51-arm-none-linux-gnueabi-i686-pc-linux-gnu.tar.bz2"
readonly CODE_SOURCERY_ROOT="${CODE_SOURCERY}/arm-2007q3"
readonly CODE_SOURCERY_BIN="${CODE_SOURCERY_ROOT}/bin"
# Make sure ${CROSS_TARGET}-* binutils are in command path
export PATH="${CODE_SOURCERY_BIN}:${PATH}"

readonly CROSS_TARGET_AS="${CODE_SOURCERY_BIN}/${CROSS_TARGET}-as"
readonly CROSS_TARGET_LD="${CODE_SOURCERY_BIN}/${CROSS_TARGET}-ld"

readonly SYSROOT="${CODE_SOURCERY_ROOT}/${CROSS_TARGET}/libc"

readonly LLVM_PKG_PATH="${LLVM_PKG_PATH:-${HOME}/llvm-project/snapshots}"

# Latest SVN revisions known to be working in this configuration.
readonly LLVM_DEFAULT_REV="74530"
readonly LLVMGCC_DEFAULT_REV="74535"

readonly LLVM_PKG="llvm-${LLVM_SVN_REV:-${LLVM_DEFAULT_REV}}.tar.bz2"
readonly LLVM_SRC_DIR="${SRC_ROOT}/llvm"
readonly LLVM_OBJ_DIR="${OBJ_ROOT}/llvm"
readonly LLVM_INSTALL_DIR="${INSTALL_ROOT}/${CROSS_TARGET}/llvm"

readonly LLVMGCC_PKG="llvm-gcc-4.2-${LLVMGCC_SVN_REV:-${LLVMGCC_DEFAULT_REV}}.tar.bz2"
readonly LLVMGCC_SRC_DIR="${SRC_ROOT}/llvm-gcc-4.2"
readonly LLVMGCC_OBJ_DIR="${OBJ_ROOT}/llvm-gcc-4.2"
readonly LLVMGCC_INSTALL_DIR="${INSTALL_ROOT}/${CROSS_TARGET}/llvm-gcc-4.2"

readonly MAKE_OPTS="${MAKE_OPTS:--j2}"

# Params:
#   $1: directory to be created
#   $2: optional mkdir command prefix, e.g. "sudo"
createDir() {
  if [[ ! -e $1 ]]; then
    ${2:-} mkdir -p $1
  elif [[ -e $1 && ! -d $1 ]]; then
    echo "$1 exists but is not a directory; exiting."
    exit 3
  fi
}

sudoCreateDir() {
  createDir $1 sudo
  sudo chown ${USER} $1
}

# Prints out and runs the command, but without logging -- intended for use with
# lightweight commands that don't have useful output to parse, e.g. mkdir, tar,
# etc.
{'fraction_non_alphanumeric': 0.14146668904178553, 'fraction_numerical': 0.021983554287632153, 'mean_word_length': 5.757369614512472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3619923', 'n_tokens_mistral': 2416, 'n_tokens_neox': 2175, 'n_words': 474}