<commit_before>#!/bin/sh
#
# Create the snappy-spark symlink in top-level snappy-commons directory.
#

# Get the parent base directory of this script
scriptdir="`dirname "$0"`"
basedir="`cd "${scriptdir}/.." && pwd`"

if [ ! -h "${basedir}/snappy-spark" ]; then
  [ -e "${basedir}/snappy-spark" ] && echo "${basedir}/snappy-spark exists but is not a symlink" && exit 1

  sspdir="${basedir}/../snappy-spark"
  # Search for snappy-spark first in SPARK_HOME
  if [ -n "${SPARK_HOME}" ]; then
    ln -s "${SPARK_HOME}" "${basedir}"
  # Then one level up
  elif [ -d "${sspdir}" ]; then
    ln -s ../snappy-spark "${basedir}"
  else
    echo "Failed to find ${basedir}/../snappy-spark. Either set SPARK_HOME to its location or place it in the same directory as snappy-commons."
    exit 1
  fi
fi

exit 0<commit_msg>added a "realpath" function and use that consistently in the script to check path equality etc<commit_after>#!/bin/sh
#
# Create the snappy-spark symlink in top-level snappy-commons directory.
#

# Get the parent base directory of this script
scriptdir="`dirname "$0"`"

realpath() {
  ( cd "$1" && pwd )
}

basedir="`realpath "${scriptdir}/.."`"
destdir="${basedir}/snappy-spark"

if [ ! -d "${destdir}" ]; then
  [ -e "${destdir}" ] && echo "${destdir} exists but is not a directory" && exit 1

  sspdir="${basedir}/../snappy-spark"
  # Search for snappy-spark first in SPARK_HOME
  if [ -n "${SPARK_HOME}" -a -d "${SPARK_HOME}" ]; then
    if [ "`realpath "${SPARK_HOME}"`" != "${destdir}" ]; then
      cp -a "${SPARK_HOME}" "${basedir}"
    fi
  # Then one level up
  elif [ -d "${sspdir}" ]; then
    sspdir="`realpath "${sspdir}"`"
    cp -a "${sspdir}" "${destdir}"
  else
    echo "Failed to find ${sspdir}. Either set SPARK_HOME to its location or place it in the same directory as snappy-commons."
    exit 1
  fi
fi

exit 0
{'fraction_non_alphanumeric': 0.17271250676773145, 'fraction_numerical': 0.004872766648619383, 'mean_word_length': 3.4745762711864407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5730416', 'n_tokens_mistral': 663, 'n_tokens_neox': 634, 'n_words': 213}