<commit_before>#/bin/bash

# Requires ENV Variable AP03_SPP to be set.

# Expected call format
#
# $: mkdir -p ~/tmp
# $: rm -rfd  ~/tmp/*
# $: AP03_SPP "MAGPIES"
# $: export AP03_SPP
# $: qsub -S /bin/bash -V -o ~/tmp/my_std_out -e ~/tmp/my_std_err modelspp.sh
#
# -V means pass all env variables from the caller's env through to the script env.
# -S specifies the shell type
# -o/-e specifies where the stdout and stderr will be written

# WORKING_DIR DIR is set by ENV variables

# Determine path to config (based on path to this file)
CONFIG_DIR="$WORKING_DIR/config"
ENVIRONMENT_CFG="$CONFIG_DIR/environment.cfg"

# Load the config file.
# This defines file paths, etc.
#
# Relevant vars set:
#   MAXENT
#   TRAINCLIMATE
#   PROJECTCLIMATE
source "$ENVIRONMENT_CFG"

mkdir -p $WORKING_DIR

# Define the species
# The species is expected to be provided as an environmental variable
if [ -z "$AP03_SPP" ]; then
    echo "SPP env variable (AP03_SPP) required. Please set the SPP env variable (AP03_SPP) and try again" 1>&2
    exit 1
fi

SPP=$AP03_SPP

# Determine the occurrences file.
# Use the private file if it exists, fall back to the public file if it doesn't
#
# The private file will contain unobfiscated data,
# the public file may contain obfiscated data.
PUBLIC_OCCUR="$WORKING_DIR/inputs/$SPP/public_occur.csv"
PRIVATE_OCCUR="$WORKING_DIR/inputs/$SPP/.private_occur.csv"

OCCUR=""

if [ -f "$PRIVATE_OCCUR" ]; then
    OCCUR=$PRIVATE_OCCUR
elif [ -f "$PUBLIC_OCCUR" ]; then
    OCCUR=$PUBLIC_OCCUR
else
    echo "No occurrences file found for $SPP" 1>&2
    exit 2
fi

# Load the java module for the HPC
module load java

# Move to the species directory
cd "$WORKING_DIR/inputs/$SPP"

# Make an output directory
TMP_OUTPUT_DIR="$WORKING_DIR/tmp_outputs/$SPP"
FINAL_OUTPUT_DIR="$WORKING_DIR/outputs/$SPP"

mkdir -p "$TMP_OUTPUT_DIR"
mkdir -p "$FINAL_OUTPUT_DIR"
<commit_msg>Changed how the outputs folder is generated.<commit_after>#/bin/bash

# Requires ENV Variable AP03_SPP to be set.

# Expected call format
#
# $: mkdir -p ~/tmp
# $: rm -rfd  ~/tmp/*
# $: AP03_SPP "MAGPIES"
# $: export AP03_SPP
# $: qsub -S /bin/bash -V -o ~/tmp/my_std_out -e ~/tmp/my_std_err modelspp.sh
#
# -V means pass all env variables from the caller's env through to the script env.
# -S specifies the shell type
# -o/-e specifies where the stdout and stderr will be written

# WORKING_DIR DIR is set by ENV variables

# Determine path to config (based on path to this file)
CONFIG_DIR="$WORKING_DIR/config"
ENVIRONMENT_CFG="$CONFIG_DIR/environment.cfg"

# Load the config file.
# This defines file paths, etc.
#
# Relevant vars set:
#   MAXENT
#   TRAINCLIMATE
#   PROJECTCLIMATE
source "$ENVIRONMENT_CFG"

mkdir -p $WORKING_DIR

# Define the species
# The species is expected to be provided as an environmental variable
if [ -z "$AP03_SPP" ]; then
    echo "SPP env variable (AP03_SPP) required. Please set the SPP env variable (AP03_SPP) and try again" 1>&2
    exit 1
fi

SPP=$AP03_SPP

# Determine the occurrences file.
# Use the private file if it exists, fall back to the public file if it doesn't
#
# The private file will contain unobfiscated data,
# the public file may contain obfiscated data.
PUBLIC_OCCUR="$WORKING_DIR/inputs/$SPP/public_occur.csv"
PRIVATE_OCCUR="$WORKING_DIR/inputs/$SPP/.private_occur.csv"

OCCUR=""

if [ -f "$PRIVATE_OCCUR" ]; then
    OCCUR=$PRIVATE_OCCUR
elif [ -f "$PUBLIC_OCCUR" ]; then
    OCCUR=$PUBLIC_OCCUR
else
    echo "No occurrences file found for $SPP" 1>&2
    exit 2
fi

# Load the java module for the HPC
module load java

# Move to the species directory
cd "$WORKING_DIR/inputs/$SPP"

# Make an output directory
TMP_OUTPUT_DIR="$WORKING_DIR/tmp_outputs/$SPP"
OUTPUTS_DIR="$WORKING_DIR/outputs"
FINAL_OUTPUT_DIR="$OUTPUTS_DIR/$SPP"

mkdir -p "$TMP_OUTPUT_DIR"
mkdir -p "$OUTPUTS_DIR"

{'fraction_non_alphanumeric': 0.10368242360929747, 'fraction_numerical': 0.010446591799425437, 'mean_word_length': 4.140939597315437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6405729', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1379, 'n_words': 459}