<commit_before>#!/bin/bash

###
# usage

# every file in the TCGA MAF project should begin
# by sourcing tcga_maf_util.sh, NOT this file


###
# description

# This file defines global variables, defines functions,
# and then runs tests and set global variables


###
# global variables
#
# to avoid unnecessary re-execution, any global variables
# must be eventually exported, so that they will be picked up
# by any subshell.
#
# TODO: revisit this if there are any functions that a subshell
# might actually need to use.

SCRIPT_PATH=
TCGAMAF_IMPORTED_UTILS=



###
# functions


# TODO more helpful error messages

# examples for currently filesystem layout:

# TGCAMAF_DATA_DIR=$TCAGMAF_ROOT_DIR/data
# TGCAMAF_REFERENCES_DIR=$TCAGMAF_ROOT_DIR/reference
# TGCAMAF_TOOLS_DIR=$TCAGMAF_ROOT_DIR/tools
# TGCAMAF_SCRIPTS_DIR=$TCAGMAF_ROOT_DIR/src
verifyRequiredVariables () {
    : ${LD_LIBRARY_PATH:?" environment variable must be set and non-empty"}

    # path to #TODO
    : ${TCGAMAF_ROOT_DIR:?" environment variable must be set and non-empty"}

    # path to #TODO
    : ${TCGAMAF_DATA_DIR:?" environment variable must be set and non-empty"}

    # TODO
    : ${TCGAMAF_REFERENCES_DIR:?" environment variable must be set and non-empty"}

    # TODO
    : ${TCGAMAF_TOOLS_DIR:?" environment variable must be set and non-empty"}

    # TODO
    : ${TCGAMAF_SCRIPTS_DIR:?" environment variable must be set and non-empty"}

    # TODO
    : ${TCGAMAF_PYTHON_BINARY:?" environment variable must be set and non-empty"}
}


# remove any symlinks by recursively expanding path, in a POSIX-compliant way
# from # http://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself
getScriptPath() {
    pushd . > /dev/null
    SCRIPT_PATH="${BASH_SOURCE[0]}";
    while([ -h "${SCRIPT_PATH}" ]); do
        cd "`dirname "${SCRIPT_PATH}"`"
        SCRIPT_PATH="$(readlink "`basename "${SCRIPT_PATH}"`")";
    done
    cd "`dirname "${SCRIPT_PATH}"`" > /dev/null
    SCRIPT_PATH="`pwd`";
    popd  > /dev/null
    export SCRIPT_PATH
}


# check that PYTHONPATH is set correctly
# TODO: the currently check only verifies that some version of
# gidget is in the path; to be more accurate, the test should make sure that
# the specific version in TCGAMAF_ROOT_DIR is being used
checkPythonPath() {
    if [[ "$PYTHONPATH" != *"gidget"* ]]; then
        echo " "
        echo " your PYTHONPATH should include paths to gidget/commands/... directories "
        echo " "
        exit 99
    fi
}



###
# run tests and set global variables

verifyRequiredVariables
getScriptPath
checkPythonPath

# this must be the last line; prevents re-execution of this script.
export TCGAMAF_IMPORTED_UTILS=IMPORTED
<commit_msg>updating required variables and bash utility checking code to use a configuration file, for easier maintence and changes, along with more friendly usage/error messages.<commit_after>#!/bin/bash

###
# usage

# every file in the TCGA MAF processing project should begin
# by sourcing tcga_maf_util.sh, NOT this file


###
# description

# This file defines global variables, defines functions,
# and then runs tests and set global variables


###
# global variables
#
# to avoid unnecessary re-execution, any global variables
# must be eventually exported, so that they will be picked up
# by any subshell.
#
# TODO: revisit this if there are any functions that a subshell
# might actually need to use.

SCRIPT_PATH=
TCGAMAF_IMPORTED_UTILS=



###
# functions


# remove any symlinks by recursively expanding path, in a POSIX-compliant way
# from # http://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself
getScriptPath() {
    pushd . > /dev/null
    SCRIPT_PATH="${BASH_SOURCE[0]}";
    while([ -h "${SCRIPT_PATH}" ]); do
        cd "`dirname "${SCRIPT_PATH}"`"
        SCRIPT_PATH="$(readlink "`basename "${SCRIPT_PATH}"`")";
    done
    cd "`dirname "${SCRIPT_PATH}"`" > /dev/null
    SCRIPT_PATH="`pwd`";
    popd  > /dev/null
    export SCRIPT_PATH
}


# checks for required variables, as described by the file named
# in the parameter 
verifyRequiredVariables () {
    while read line; do

        if [[ $line == \#* ]]; then
            # comment line
            continue
        fi

        if [ -z "$line" ]; then
            # blank line
            continue
        fi

        # the $'\t' syntax below is an 'ANSI-C' bash string:
        # http://stackoverflow.com/a/6656767
        varName=${line%%$'\t'*}
        descr=${line##*$'\t'}
        # echo [$varName]: $descr

        # the ${!varName} syntax is an indirect refernce,
        # supported since bash v2
        if [ -z ${!varName} ]; then
            echo "*** error: required environmental variable $varName was not set"
            echo "usage:"
            echo "  $varName: $descr"
            echo
            echo "*** exiting"
            exit -1
        fi
    done < $1
}


# check that PYTHONPATH is set correctly
# TODO: the currently check only verifies that some version of
# gidget is in the path; to be more accurate, the test should make sure that
# the specific version in TCGAMAF_ROOT_DIR is being used
checkPythonPath() {
    if [[ "$PYTHONPATH" != *"gidget"* ]]; then
        echo " "
        echo " your PYTHONPATH should include paths to gidget/commands/... directories "
        echo " "
        exit 99
    fi
}



###
# run tests and set global variables

getScriptPath
verifyRequiredVariables ${SCRIPT_PATH}/../config/required_env_vars
checkPythonPath

# this must be the last line; prevents re-execution of this script.
export TCGAMAF_IMPORTED_UTILS=IMPORTED

{'fraction_non_alphanumeric': 0.10513274336283186, 'fraction_numerical': 0.005309734513274336, 'mean_word_length': 3.1952487008166295, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3827520', 'n_tokens_mistral': 1854, 'n_tokens_neox': 1742, 'n_words': 575}