<commit_before>#!/bin/bash -l

# -----------------------------------------------------------------------------
# Author: <NAME> (<EMAIL>)
# Copyright 2013-2014, <EMAIL>
# License under the MIT License
#
# This script launches a radical.pilot compute pilot.
#

# -----------------------------------------------------------------------------
# global variables
#
VERSION=
PREBOOTSTRAP=
CLEANUP=
REMOTE=
CORES=
RUNTIME=
DBNAME=
PILOTID=
UNITMANAGERID=
SESSIONID=
WORKDIR=`pwd`
PYTHON=`which python`
LAUNCH_MODE=SSH
QUEUE=
ALLOCATION=

# -----------------------------------------------------------------------------
# print out script usage help
#
usage()
{
cat << EOF
usage: $0 options

This script launches a RADICAL-Pilot agent.

OPTIONS:
   -r      Address and port of the coordination service host (MongoDB)

   -d      The name of the database 

   -s      The unique identifier (uid) of the session

   -p      The unique identifier (uid) of the pilot

   -w      The working (base) directory of the pilot
           (default is '.')

   -i      The Python interpreter to use, e.g., python2.6
           (default is '/usr/bin/python')

   -l      Task lauch mode, AUTO, LOCAL, MPIRUN, APRUN

   -e      List of commands to run before botstrapping

   -t      Runtime in minutes

   -c      Number of requested cores

   -q      The name of the queue to use

   -a      The name of project / allocation to charge

   -C      Cleanup - delete virtualenv after execution

   -V      Version - the RADICAL-Pilot package version

   -h      Show this message

EOF
}

# -----------------------------------------------------------------------------
# bootstrap virtualenv - we always use the latest version from GitHub
#
installvenv()
{
R_SYS_DIR=$WORKDIR/virtualenv/
# remove any old versionsion
if [ -d $R_SYS_DIR ] 
then
    echo "`date +"%m-%d-%Y %T"` - [run-radical-agent.sh] (INFO) - Removing previous virtualenv: $R_SYS_DIR"
    rm -r $R_SYS_DIR
fi

# create a fresh virtualenv. we use and older 1.9.x version of 
# virtualenv as this seems to work more reliable than newer versions.
CURL_CMD="curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.tar.gz"
echo ""
echo "################################################################################"
echo "## Downloading and installing virtualenv"
echo "## CMDLINE: $CURL_CMD"
curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.tar.gz
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't download virtuelenv via curl! ABORTING"
   exit 1
fi

tar xvfz virtualenv-1.9.tar.gz
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't unpack virtualenv! ABORTING"
   exit 1
fi

BOOTSTRAP_CMD="$PYTHON virtualenv-1.9/virtualenv.py --python=$PYTHON $R_SYS_DIR"
echo ""
echo "################################################################################"
echo "## Creating virtualenv"
echo "## CMDLINE: $BOOTSTRAP_CMD"
$BOOTSTRAP_CMD
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't bootstrap virtuelenv! ABORTING"
   exit 1
fi

# active the virtualenv
source $R_SYS_DIR/bin/activate

UPDATE_SETUPTOOLS_CMD="pip install --upgrade setuptools"
echo ""
echo "################################################################################"
echo "## Updating virtualenv"
echo "## CMDLINE: $UPDATE_SETUPTOOLS_CMD"
$UPDATE_SETUPTOOLS_CMD
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't update virtualenv! ABORTING"
   exit 1
fi

# for now we use development version directly from git
# this blows up the virtualenv (and agent bootstrap time) significantly
# at some point we will bootstrap everything directly from PyPi.
# maybe a flag would be good to switch between 'production' and 'deve' ? 
PIP_CMD="pip install python-hostlist"
echo ""
echo "################################################################################"
echo "## Installing python-hostlist"
echo "## CMDLINE: $PIP_CMD"
$PIP_CMD
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't install python-hostlist! ABORTING"
   exit 1
fi

PIP_CMD="pip install pymongo"
echo ""
echo "################################################################################"
echo "## Installing pymongo"
echo "## CMDLINE: $PIP_CMD"
$PIP_CMD<commit_msg>removed setuptools update from default bootstrapper<commit_after>#!/bin/bash -l

# -----------------------------------------------------------------------------
# Author: <NAME> (<EMAIL>)
# Copyright 2013-2014, <EMAIL>
# License under the MIT License
#
# This script launches a radical.pilot compute pilot.
#

# -----------------------------------------------------------------------------
# global variables
#
VERSION=
PREBOOTSTRAP=
CLEANUP=
REMOTE=
CORES=
RUNTIME=
DBNAME=
PILOTID=
UNITMANAGERID=
SESSIONID=
WORKDIR=`pwd`
PYTHON=`which python`
LAUNCH_MODE=SSH
QUEUE=
ALLOCATION=

# -----------------------------------------------------------------------------
# print out script usage help
#
usage()
{
cat << EOF
usage: $0 options

This script launches a RADICAL-Pilot agent.

OPTIONS:
   -r      Address and port of the coordination service host (MongoDB)

   -d      The name of the database 

   -s      The unique identifier (uid) of the session

   -p      The unique identifier (uid) of the pilot

   -w      The working (base) directory of the pilot
           (default is '.')

   -i      The Python interpreter to use, e.g., python2.6
           (default is '/usr/bin/python')

   -l      Task lauch mode, AUTO, LOCAL, MPIRUN, APRUN

   -e      List of commands to run before botstrapping

   -t      Runtime in minutes

   -c      Number of requested cores

   -q      The name of the queue to use

   -a      The name of project / allocation to charge

   -C      Cleanup - delete virtualenv after execution

   -V      Version - the RADICAL-Pilot package version

   -h      Show this message

EOF
}

# -----------------------------------------------------------------------------
# bootstrap virtualenv - we always use the latest version from GitHub
#
installvenv()
{
R_SYS_DIR=$WORKDIR/virtualenv/
# remove any old versionsion
if [ -d $R_SYS_DIR ] 
then
    echo "`date +"%m-%d-%Y %T"` - [run-radical-agent.sh] (INFO) - Removing previous virtualenv: $R_SYS_DIR"
    rm -r $R_SYS_DIR
fi

# create a fresh virtualenv. we use and older 1.9.x version of 
# virtualenv as this seems to work more reliable than newer versions.
CURL_CMD="curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.tar.gz"
echo ""
echo "################################################################################"
echo "## Downloading and installing virtualenv"
echo "## CMDLINE: $CURL_CMD"
curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.tar.gz
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't download virtuelenv via curl! ABORTING"
   exit 1
fi

tar xvfz virtualenv-1.9.tar.gz
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't unpack virtualenv! ABORTING"
   exit 1
fi

BOOTSTRAP_CMD="$PYTHON virtualenv-1.9/virtualenv.py --python=$PYTHON $R_SYS_DIR"
echo ""
echo "################################################################################"
echo "## Creating virtualenv"
echo "## CMDLINE: $BOOTSTRAP_CMD"
$BOOTSTRAP_CMD
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't bootstrap virtuelenv! ABORTING"
   exit 1
fi

# active the virtualenv
source $R_SYS_DIR/bin/activate

DOWNGRADE_PIP_CMD="easy_install pip==1.2.1"
echo ""
echo "################################################################################"
echo "## Downgrading pip to 1.2.1"
echo "## CMDLINE: $DOWNGRADE_PIP_CMD"
$DOWNGRADE_PIP_CMD
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't downgrade pip! ABORTING"
   exit 1
fi

#UPDATE_SETUPTOOLS_CMD="pip install --upgrade setuptools"
#echo ""
#echo "################################################################################"
#echo "## Updating virtualenv"
#echo "## CMDLINE: $UPDATE_SETUPTOOLS_CMD"
#$UPDATE_SETUPTOOLS_CMD
#OUT=$?
#if [ $OUT -ne 0 ];then
#   echo "Couldn't update virtualenv! ABORTING"
#   exit 1
#fi

# for now we use development version directly from git
# this blows up the virtualenv (and agent bootstrap time) significantly
# at some point we will bootstrap everything directly from PyPi.
# maybe a flag would be good to switch between 'production' and 'deve' ? 
PIP_CMD="pip install python-hostlist"
echo ""
echo "################################################################################"
echo "## Installing python-hostlist"
echo "## CMDLINE: $PIP_CMD"
$PIP_CMD
OUT=$?
if [ $OUT -ne 0 ];then
   echo "Couldn't install python-hostlist! ABORTING"
   exit 1
fi

PIP_CMD="pip install pymongo"
echo ""
echo "################################################################################"
echo "## Installing pymongo"
echo "## CMDLINE: $PIP_CMD"
$PIP_CMD
{'fraction_non_alphanumeric': 0.2624885426214482, 'fraction_numerical': 0.008020164986251146, 'mean_word_length': 2.8899286987522284, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '412367', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2581, 'n_words': 851}