<commit_before>#!/bin/bash

set -e
# no "set -u" because installation scripts address various undefined variables

# Version comparison functions
function verlte() {
    [  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
}

function verlt() {
    [ "$1" = "$2" ] && return 1 || verlte "$1" "$2"
}

# TODO: test python 3.6
PYTHON3_MAJOR_VERSION="3.5"

echo "Installing (upgrading) Pip..."
if [ `uname` == 'Darwin' ]; then
    # doesn't need get-pip as python always comes with pip installed on Mac OS X
    # should theoretically be pre-installed on all platforms from Python 2.7+
    pip2.7 install --upgrade pip
    pip$PYTHON3_MAJOR_VERSION install --upgrade pip
else
    # assume Ubuntu
    wget https://bootstrap.pypa.io/get-pip.py -O - | $COMMAND_PREFIX python2.7 -
    sudo rm setuptools-*.zip || echo "No setuptools to cleanup"
    wget https://bootstrap.pypa.io/get-pip.py -O - | $COMMAND_PREFIX python$PYTHON3_MAJOR_VERSION -
    sudo rm setuptools-*.zip || echo "No setuptools to cleanup"
fi

echo "Installing (upgrading) Supervisor..."
# * change dir, otherwise the installer might think we're trying to install from the supervisor/ directory
# * also, Supervisor only supports Python 2.7 at the moment
( cd /tmp; $COMMAND_PREFIX pip2.7 install --upgrade supervisor )

echo "Installing (upgrading) Virtualenv..."
$COMMAND_PREFIX pip2.7 install --force-reinstall --upgrade virtualenv
$COMMAND_PREFIX pip$PYTHON3_MAJOR_VERSION install --force-reinstall --upgrade virtualenv

echo "Creating mc-venv virtualenv..."
echo "$(which python$PYTHON3_MAJOR_VERSION)"
echo "$(which virtualenv)"
virtualenv --python=python$PYTHON3_MAJOR_VERSION mc-venv
source mc-venv/bin/activate

echo "Adding 'mediacloud/' to module search path..."<commit_msg>fix missing sudo when running get-pip on ubuntu<commit_after>#!/bin/bash

set -e
# no "set -u" because installation scripts address various undefined variables

# Version comparison functions
function verlte() {
    [  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
}

function verlt() {
    [ "$1" = "$2" ] && return 1 || verlte "$1" "$2"
}

# TODO: test python 3.6
PYTHON3_MAJOR_VERSION="3.5"

echo "Installing (upgrading) Pip..."
if [ `uname` == 'Darwin' ]; then
    # doesn't need get-pip as python always comes with pip installed on Mac OS X
    # should theoretically be pre-installed on all platforms from Python 2.7+
    pip2.7 install --upgrade pip
    pip$PYTHON3_MAJOR_VERSION install --upgrade pip
else
    # assume Ubuntu
    wget https://bootstrap.pypa.io/get-pip.py -O - | sudo python2.7 -
    sudo rm setuptools-*.zip || echo "No setuptools to cleanup"
    wget https://bootstrap.pypa.io/get-pip.py -O - | sudo python$PYTHON3_MAJOR_VERSION -
    sudo rm setuptools-*.zip || echo "No setuptools to cleanup"
fi

echo "Installing (upgrading) Supervisor..."
# * change dir, otherwise the installer might think we're trying to install from the supervisor/ directory
# * also, Supervisor only supports Python 2.7 at the moment
( cd /tmp; $COMMAND_PREFIX pip2.7 install --upgrade supervisor )

echo "Installing (upgrading) Virtualenv..."
$COMMAND_PREFIX pip2.7 install --force-reinstall --upgrade virtualenv
$COMMAND_PREFIX pip$PYTHON3_MAJOR_VERSION install --force-reinstall --upgrade virtualenv

echo "Creating mc-venv virtualenv..."
echo "$(which python$PYTHON3_MAJOR_VERSION)"
echo "$(which virtualenv)"
virtualenv --python=python$PYTHON3_MAJOR_VERSION mc-venv
source mc-venv/bin/activate

echo "Adding 'mediacloud/' to module search path..."
{'fraction_non_alphanumeric': 0.12350256702795208, 'fraction_numerical': 0.017683970336565887, 'mean_word_length': 3.8640776699029127, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4808524', 'n_tokens_mistral': 1208, 'n_tokens_neox': 1151, 'n_words': 372}