<commit_before>#!/bin/bash

set -u
set -e

source ./config.sh

# Remove and then recreate temporary directory
if [ -d ${KMD_TMP_DIR} ]; then
  rm -rf ${KMD_TMP_DIR}
fi

mkdir ${KMD_TMP_DIR}

# 1. Clone config files so we have up to date versions
# Only clone the repository if it does not exist on disk
if [ ! -d ${CONFIG_SRC_DIR} ]; then
  cd ${KMD_TMP_DIR}
  git clone ${CONFIG_URL}
fi

# 2. Download and build glib
# Only download the tarball if it does not exist
if [ ! -f ${GLIB_TARBALL} ]; then
  wget ${GLIB_URL} -O ${GLIB_TARBALL}
fi

# Remove the source directory if it exists
if [ -d ${GLIB_SRC_DIR} ]; then
  rm -rf ${GLIB_SRC_DIR}
fi

cd ${KMD_TMP_DIR}
tar xf ${GLIB_TARBALL}

# Copy up to date versions of config files
cp ${CONFIG_SRC_DIR}/config.sub ${GLIB_SRC_DIR}/config.sub
cp ${CONFIG_SRC_DIR}/config.guess ${GLIB_SRC_DIR}/config.guess

# Configure and build glib
cd ${GLIB_SRC_DIR}
./configure ${GLIB_CONFIGURE_OPTIONS[*]}
make
${SUDO_CMD} make install

# 3. Download and build GTK
# Only download the tarball if it does not exist
if [ ! -f ${GTK_TARBALL} ]; then
  wget ${GTK_URL} -O ${GTK_TARBALL}
fi

# Remove the source directory if it exists
if [ -d ${GTK_SRC_DIR} ]; then
  rm -rf ${GTK_SRC_DIR}
fi

cd ${KMD_TMP_DIR}
tar xf ${GTK_TARBALL}

# Copy up to date versions of config files
cp ${CONFIG_SRC_DIR}/config.sub ${GTK_SRC_DIR}/config.sub
cp ${CONFIG_SRC_DIR}/config.guess ${GTK_SRC_DIR}/config.guess

# Configure and build GTK
cd ${GTK_SRC_DIR}
./configure ${GTK_CONFIGURE_OPTIONS[*]}
make
${SUDO_CMD} make install

# 4. Download and build Komodo
# Only download the tarball if it does not exist
if [ ! -f ${KMD_TARBALL} ]; then
  wget ${KMD_URL} -O ${KMD_TARBALL}
fi

# Remove the source directory if it exists
if [ -d ${KMD_SRC_DIR} ]; then
  rm -rf ${KMD_SRC_DIR}
fi

cd ${KMD_TMP_DIR}
tar xf ${KMD_TARBALL}

# Copy up to date versions of config files
cp ${CONFIG_SRC_DIR}/config.sub ${KMD_SRC_DIR}/config.sub
cp ${CONFIG_SRC_DIR}/config.guess ${KMD_SRC_DIR}/config.guess

# Set up PATH so it can find GTK config
export PATH="${PATH}:${KMD_TMP_DIR}/bin"

# Configure and build glib
cd ${KMD_SRC_DIR}
./configure ${KMD_CONFIGURE_OPTIONS[*]}
make
${SUDO_CMD} make install
<commit_msg>Hack to make install-sh work on OS X<commit_after>#!/bin/bash

set -u
set -e

source ./config.sh

# Remove and then recreate temporary directory
if [ -d ${KMD_TMP_DIR} ]; then
  rm -rf ${KMD_TMP_DIR}
fi

mkdir ${KMD_TMP_DIR}

# 1. Clone config files so we have up to date versions
# Only clone the repository if it does not exist on disk
if [ ! -d ${CONFIG_SRC_DIR} ]; then
  cd ${KMD_TMP_DIR}
  git clone ${CONFIG_URL}
fi

# 2. Download and build glib
# Only download the tarball if it does not exist
if [ ! -f ${GLIB_TARBALL} ]; then
  wget ${GLIB_URL} -O ${GLIB_TARBALL}
fi

# Remove the source directory if it exists
if [ -d ${GLIB_SRC_DIR} ]; then
  rm -rf ${GLIB_SRC_DIR}
fi

cd ${KMD_TMP_DIR}
tar xf ${GLIB_TARBALL}

# Copy up to date versions of config files
cp ${CONFIG_SRC_DIR}/config.sub ${GLIB_SRC_DIR}/config.sub
cp ${CONFIG_SRC_DIR}/config.guess ${GLIB_SRC_DIR}/config.guess

# Configure and build glib
cd ${GLIB_SRC_DIR}
./configure ${GLIB_CONFIGURE_OPTIONS[*]}
make
${SUDO_CMD} make install

# 3. Download and build GTK
# Only download the tarball if it does not exist
if [ ! -f ${GTK_TARBALL} ]; then
  wget ${GTK_URL} -O ${GTK_TARBALL}
fi

# Remove the source directory if it exists
if [ -d ${GTK_SRC_DIR} ]; then
  rm -rf ${GTK_SRC_DIR}
fi

cd ${KMD_TMP_DIR}
tar xf ${GTK_TARBALL}

# Copy up to date versions of config files
cp ${CONFIG_SRC_DIR}/config.sub ${GTK_SRC_DIR}/config.sub
cp ${CONFIG_SRC_DIR}/config.guess ${GTK_SRC_DIR}/config.guess

# Configure and build GTK
cd ${GTK_SRC_DIR}
./configure ${GTK_CONFIGURE_OPTIONS[*]}
make
${SUDO_CMD} make install

# 4. Download and build Komodo
# Only download the tarball if it does not exist
if [ ! -f ${KMD_TARBALL} ]; then
  wget ${KMD_URL} -O ${KMD_TARBALL}
fi

# Remove the source directory if it exists
if [ -d ${KMD_SRC_DIR} ]; then
  rm -rf ${KMD_SRC_DIR}
fi

cd ${KMD_TMP_DIR}
tar xf ${KMD_TARBALL}

# Copy up to date versions of config files
cp ${CONFIG_SRC_DIR}/config.sub ${KMD_SRC_DIR}/config.sub
cp ${CONFIG_SRC_DIR}/config.guess ${KMD_SRC_DIR}/config.guess

# Set up PATH so it can find GTK config
export PATH="${PATH}:${KMD_TMP_DIR}/bin"

# Configure and build Komodo
cd ${KMD_SRC_DIR}
./configure ${KMD_CONFIGURE_OPTIONS[*]}
make

# Horrible hack to get around the problem of OS X refusing to execute
# a script, even if run via /bin/sh install-sh
chmod 755 ${KMD_SRC_DIR}/install-sh

${SUDO_CMD} make install

{'fraction_non_alphanumeric': 0.12065972222222222, 'fraction_numerical': 0.002387152777777778, 'mean_word_length': 4.371794871794871, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5943526', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1800, 'n_words': 577}