<commit_before>#!/bin/bash

# <NAME> 20190522

print_help() {
cat <<EOF
Run a Jupyter notebook on Gadi's compute nodes, presenting the interface in a
browser on the local machine
General Options:
    -h:         Print help
    -l:         Gadi username
    -L:         Gadi login node (default 'gadi.nci.org.au')
Queue Options:
    -q QUEUE:   Queue name
    -n NCPU:    Use NCPU cpus
    -m MEM:     Memory allocation (default 2*NCPU GB)
    -t TIME:    Walltime limit (default 1 hour)
    -J JOBFS:   Jobfs allocation (default 100 GB)
    -P PROJ:    Submit job under project PROJ
EOF
}

set -eu

# Internal defaults
USER=''
PROJECT='-P v45' # Note- should be the full pbs flag '-P a12' if overriding
LOGINNODE='gadi.nci.org.au'
QUEUE='express'
NCPUS='48'
MEM='192gb'
WALLTIME=1:00:00
JOBFS=100gb

# Handle arguments
optspec="hl:L:q:n:m:t:J:P:"
while getopts "$optspec" optchar; do
    case "${optchar}" in
        h)
            print_help
            exit 2
            ;;
        l)
            USER="${OPTARG}"
            ;;
        L)
            LOGINNODE="${OPTARG}"
            ;;
        q)
            QUEUE="${OPTARG}"
            ;;
        n)
            NCPUS="${OPTARG}"
            ;;
        m)
            MEM="${OPTARG}"
            ;;
        t)
            WALLTIME="${OPTARG}"
            ;;
        J)
            JOBFS="${OPTARG}"
            ;;
        P)
            PROJECT="-P '${OPTARG}'"
            ;;
        *)
            print_help
            exit 2
            ;;
    esac
done

# This gets evaluated on Gadi in the SSH script
WORKDIR=\$TMPDIR/runjp

SSH='ssh -oBatchMode=yes'
if [ -n "$USER" ]; then
    SSH="${SSH} -l ${USER}"
fi
if [ -z "$MEM" ]; then
    MEM="$(( NCPUS * 2 ))gb"
fi

SUBMITOPTS="-N jupyter-notebook $PROJECT -q '$QUEUE' -l 'ncpus=${NCPUS},mem=${MEM},walltime=${WALLTIME},jobfs=${JOBFS}'"

echo "Starting notebook on ${LOGINNODE}..."<commit_msg>lower default core count on jupyter_gadi (#160)<commit_after>#!/bin/bash

# <NAME> 20190522

print_help() {
cat <<EOF
Run a Jupyter notebook on Gadi's compute nodes, presenting the interface in a
browser on the local machine
General Options:
    -h:         Print help
    -l:         Gadi username
    -L:         Gadi login node (default 'gadi.nci.org.au')
Queue Options:
    -q QUEUE:   Queue name
    -n NCPU:    Use NCPU cpus
    -m MEM:     Memory allocation (default 4*NCPU GB)
    -t TIME:    Walltime limit (default 1 hour)
    -J JOBFS:   Jobfs allocation (default 100 GB)
    -P PROJ:    Submit job under project PROJ
EOF
}

set -eu

# Internal defaults
USER=''          # Add your nci username here
PROJECT='-P v45' # Note- should be the full pbs flag '-P a12' if overriding
LOGINNODE='gadi.nci.org.au'
QUEUE='express'  # QUEUE, NCPUS and MEM can be overridden in command line
NCPUS='8'
MEM='32gb'
WALLTIME=1:00:00
JOBFS=100gb

# Handle arguments
optspec="hl:L:q:n:m:t:J:P:"
while getopts "$optspec" optchar; do
    case "${optchar}" in
        h)
            print_help
            exit 2
            ;;
        l)
            USER="${OPTARG}"
            ;;
        L)
            LOGINNODE="${OPTARG}"
            ;;
        q)
            QUEUE="${OPTARG}"
            ;;
        n)
            NCPUS="${OPTARG}"
            ;;
        m)
            MEM="${OPTARG}"
            ;;
        t)
            WALLTIME="${OPTARG}"
            ;;
        J)
            JOBFS="${OPTARG}"
            ;;
        P)
            PROJECT="-P '${OPTARG}'"
            ;;
        *)
            print_help
            exit 2
            ;;
    esac
done

# This gets evaluated on Gadi in the SSH script
WORKDIR=\$TMPDIR/runjp

SSH='ssh -oBatchMode=yes'
if [ -n "$USER" ]; then
    SSH="${SSH} -l ${USER}"
fi
if [ -z "$MEM" ]; then
    MEM="$(( NCPUS * 4 ))gb"
fi

SUBMITOPTS="-N jupyter-notebook $PROJECT -q '$QUEUE' -l 'ncpus=${NCPUS},mem=${MEM},walltime=${WALLTIME},jobfs=${JOBFS}'"

echo "Starting notebook on ${LOGINNODE}..."
{'fraction_non_alphanumeric': 0.13915693245302183, 'fraction_numerical': 0.0170137125444388, 'mean_word_length': 1.8135714285714286, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2230335', 'n_tokens_mistral': 1444, 'n_tokens_neox': 1340, 'n_words': 380}