<commit_before>#!/bin/bash
set -e
cd `dirname $0`/..

git submodule update --init --recursive

PLATFORM=`uname -s`
PKGMGR=""

if [ "$PLATFORM" = "Darwin" ]; then
    if command -v brew; then
        echo "Found the homebrew package manager."
        PKGMGR="brew install"
    fi
fi

cd vendor

# check for the JCL
if [ ! -f classes/java/lang/Object.class ]; then
  DOWNLOAD_DIR=`mktemp -d jdk-download.XXX`
  cd $DOWNLOAD_DIR
    DEBS_DOMAIN="http://security.ubuntu.com/ubuntu/pool/main/o/openjdk-6"
    DEBS=("openjdk-6-jre-headless_6b24-1.11.5-0ubuntu1~11.10.1_i386.deb"
          "openjdk-6-jdk_6b24-1.11.5-0ubuntu1~11.10.1_i386.deb"
          "openjdk-6-jre-lib_6b24-1.11.5-0ubuntu1~11.10.1_all.deb")
    for DEB in ${DEBS[@]}; do
      wget $DEBS_DOMAIN/$DEB
      ar p $DEB data.tar.gz | tar zx
    done
  cd ..
  JARS=("rt.jar" "tools.jar" "resources.jar" "rhino.jar")
  for JAR in ${JARS[@]}; do
    JAR_PATH=`find $DOWNLOAD_DIR/usr -name $JAR`
    echo "Extracting the Java class library from $JAR_PATH"
    unzip -qq -o -d classes/ "$JAR_PATH"
  done
  if [ ! -e java_home ]; then
    JH=$DOWNLOAD_DIR/usr/lib/jvm/java-6-openjdk/jre
    # a number of .properties files are symlinks to /etc; copy the targets over
    # so we do not need to depend on /etc's existence
    for LINK in `find $JH -type l`; do
      DEST=`readlink $LINK`
      if [ "`expr "$DEST" : '/etc'`" != "0" ]; then
        test -e "$DOWNLOAD_DIR/$DEST" && mv "$DOWNLOAD_DIR/$DEST" $LINK
      fi
    done
    mv $JH java_home
  fi
  rm -rf $DOWNLOAD_DIR
fi

# check for jazzlib
if [ ! -f classes/java/util/zip/DeflaterEngine.class ]; then
  echo "patching the class library with Jazzlib"
  mkdir -p jazzlib && cd jazzlib
  if ! command -v wget >/dev/null && [ -n "$PKGMGR" ]; then
      $PKGMGR wget
  fi
  wget -q "http://downloads.sourceforge.net/project/jazzlib/jazzlib/0.07/jazzlib-binary-0.07-juz.zip"
  unzip -qq "jazzlib-binary-0.07-juz.zip"

  cp java/util/zip/*.class ../classes/java/util/zip/
  cd .. && rm -rf jazzlib
fi

cd ..  # back to start

# Make sure node is present and >= v0.8
if [[ `node -v` < "v0.8" ]]; then
  echo "node >= v0.8 required"
  if [ -n "$PKGMGR" ]; then
    $PKGMGR node
  else
    exit
  fi
fi 
echo "Installing required node modules"
make dependencies

echo "Using `javac -version 2>&1` to generate classfiles"
make java

if ! command -v bundle > /dev/null; then
    if command -v gem > /dev/null; then
        gem install bundler
    else
        echo "warning: could not install bundler because rubygems was not found!"
        echo "some dependencies may be missing."
    fi
fi

command -v bundle > /dev/null && bundle install

# does sed support extended regexps?
if ! sed -r "" </dev/null >/dev/null 2>&1 && ! command -v gsed >/dev/null; then
    if [ "$PLATFORM" = "Darwin" ] && [ -n "$PKGMGR" ]; then
        $PKGMGR gnu-sed
    else
        echo "warning: sed does not seem to support extended regular expressions."
        echo "Doppio can run without this, but it is needed for building the full website."
    fi
fi

# Intentionally fail if pygmentize doesn't exist.
echo "Checking for pygment (needed to generate docs)... `pygmentize -V`"

echo "Your environment should now be set up correctly."
echo "Run 'make test' (optionally with -j4) to test Doppio."
<commit_msg>[TOOLS](\setup.sh) - Fix path in Ubuntu classload<commit_after>#!/bin/bash
set -e
cd `dirname $0`/..

git submodule update --init --recursive

PLATFORM=`uname -s`
PKGMGR=""

if [ "$PLATFORM" = "Darwin" ]; then
    if command -v brew; then
        echo "Found the homebrew package manager."
        PKGMGR="brew install"
    fi
fi

cd vendor

# check for the JCL
# Ubuntu (security) repo actual on 24.02.2013
if [ ! -f classes/java/lang/Object.class ]; then
  DOWNLOAD_DIR=`mktemp -d jdk-download.XXX`
  cd $DOWNLOAD_DIR
    DEBS_DOMAIN="http://security.ubuntu.com/ubuntu/pool/main/o/openjdk-6"
    DEBS=("openjdk-6-jre-headless_6b24-1.11.5-0ubuntu1~11.04.1_i386.deb"
          "openjdk-6-jdk_6b24-1.11.5-0ubuntu1~11.04.1_i386.deb"
          "openjdk-6-jre-lib_6b24-1.11.5-0ubuntu1~11.04.1_all.deb")
    for DEB in ${DEBS[@]}; do
      wget $DEBS_DOMAIN/$DEB
      ar p $DEB data.tar.gz | tar zx
    done
  cd ..
  JARS=("rt.jar" "tools.jar" "resources.jar" "rhino.jar")
  for JAR in ${JARS[@]}; do
    JAR_PATH=`find $DOWNLOAD_DIR/usr -name $JAR`
    echo "Extracting the Java class library from $JAR_PATH"
    unzip -qq -o -d classes/ "$JAR_PATH"
  done
  if [ ! -e java_home ]; then
    JH=$DOWNLOAD_DIR/usr/lib/jvm/java-6-openjdk/jre
    # a number of .properties files are symlinks to /etc; copy the targets over
    # so we do not need to depend on /etc's existence
    for LINK in `find $JH -type l`; do
      DEST=`readlink $LINK`
      if [ "`expr "$DEST" : '/etc'`" != "0" ]; then
        test -e "$DOWNLOAD_DIR/$DEST" && mv "$DOWNLOAD_DIR/$DEST" $LINK
      fi
    done
    mv $JH java_home
  fi
  rm -rf $DOWNLOAD_DIR
fi

# check for jazzlib
if [ ! -f classes/java/util/zip/DeflaterEngine.class ]; then
  echo "patching the class library with Jazzlib"
  mkdir -p jazzlib && cd jazzlib
  if ! command -v wget >/dev/null && [ -n "$PKGMGR" ]; then
      $PKGMGR wget
  fi
  wget -q "http://downloads.sourceforge.net/project/jazzlib/jazzlib/0.07/jazzlib-binary-0.07-juz.zip"
  unzip -qq "jazzlib-binary-0.07-juz.zip"

  cp java/util/zip/*.class ../classes/java/util/zip/
  cd .. && rm -rf jazzlib
fi

cd ..  # back to start

# Make sure node is present and >= v0.8
if [[ `node -v` < "v0.8" ]]; then
  echo "node >= v0.8 required"
  if [ -n "$PKGMGR" ]; then
    $PKGMGR node
  else
    exit
  fi
fi 
echo "Installing required node modules"
make dependencies

echo "Using `javac -version 2>&1` to generate classfiles"
make java

if ! command -v bundle > /dev/null; then
    if command -v gem > /dev/null; then
        gem install bundler
    else
        echo "warning: could not install bundler because rubygems was not found!"
        echo "some dependencies may be missing."
    fi
fi

command -v bundle > /dev/null && bundle install

# does sed support extended regexps?
if ! sed -r "" </dev/null >/dev/null 2>&1 && ! command -v gsed >/dev/null; then
    if [ "$PLATFORM" = "Darwin" ] && [ -n "$PKGMGR" ]; then
        $PKGMGR gnu-sed
    else
        echo "warning: sed does not seem to support extended regular expressions."
        echo "Doppio can run without this, but it is needed for building the full website."
    fi
fi

# Intentionally fail if pygmentize doesn't exist.
echo "Checking for pygment (needed to generate docs)... `pygmentize -V`"

echo "Your environment should now be set up correctly."
echo "Run 'make test' (optionally with -j4) to test Doppio."

{'fraction_non_alphanumeric': 0.13562874251497006, 'fraction_numerical': 0.023652694610778444, 'mean_word_length': 2.722005571030641, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5345778', 'n_tokens_mistral': 2617, 'n_tokens_neox': 2479, 'n_words': 761}