<commit_before>#!/usr/bin/env bash
set -e

#User variables
SOURCEISS3BUCKET="True"
AWSREGION="us-east-1"
AWSCLI_URL="https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"
ROOT_CERT_URL=""
SYSTEMPREPMASTERSCRIPTSOURCE="https://s3.amazonaws.com/systemprep/MasterScripts/systemprep-linuxmaster.py"
SYSTEMPREPPARAMS=( "SaltStates=Highstate"
                   "NoReboot=False"
                   "SourceIsS3Bucket=${SOURCEISS3BUCKET}"
                   "AwsRegion=${AWSREGION}" )

#System variables
SCRIPTNAME=${0}
LOGGER=$(which logger)
TIMESTAMP=$(date -u +"%Y%m%d_%H%M_%S")
WORKINGDIR=/usr/tmp
LOGDIR=/var/log
LOGTAG=systemprep
LOGFILE="${LOGDIR}/${LOGTAG}-${TIMESTAMP}.log"
LOGLINK="${LOGDIR}/${LOGTAG}.log"

# Validate log directory exists
if [[ ! -d ${LOGDIR} ]]; then
  echo "Creating ${LOGDIR} directory." >(${LOGGER} -t "${LOGTAG}" -s 2> /dev/console) 2>&1
  mkdir ${LOGDIR} >(${LOGGER} -i -t "${LOGTAG}" -s 2> /dev/console) 2>&1
fi

# Validate working directory exists
if [[ ! -d ${WORKINGDIR} ]]; then
  echo "Creating ${WORKINGDIR} directory" >(${LOGGER} -t "${LOGTAG}" -s 2> /dev/console) 2>&1
  mkdir ${WORKINGDIR} >(${LOGGER} -i -t "${LOGTAG}" -s 2> /dev/console) 2>&1
fi

# Establish logging to write to the logfile, syslog, and the console
exec > >(tee "${LOGFILE}" | "${LOGGER}" -i -t "${LOGTAG}" -s 2> /dev/console) 2>&1

# Create the link to the logfile
touch ${LOGFILE}
ln -s -f ${LOGFILE} ${LOGLINK}

# Change to the working directory
cd ${WORKINGDIR}

# Create the log file and write out the parameters
echo "Entering SystemPrep script -- ${SCRIPTNAME}"
echo "Writing SystemPrep Parameters to log file..."
for param in "${SYSTEMPREPPARAMS[@]}"; do echo "   ${param}" ; done

# Install root certs, if the root cert url is provided
if [[ -n "${ROOT_CERT_URL}" ]]; then

    #######################################################
    # Are we EL-compatible and do we have 6.5+ behaviour?
    #######################################################
    GetMode() {
        UPDATETRUST="/usr/bin/update-ca-trust"
        CERTUTIL="/usr/bin/certutil"
        if [ -x ${UPDATETRUST} ]
        then
            echo "6.5"
        elif [ -x ${CERTUTIL} ]
        then
            echo "6.0"
        else
            echo "Cannot determine CA update-method. Aborting."
            exit 1
        fi
    }

    #########################################################
    # Try to fetch all CA .cer files from our root_cert_url
    #########################################################
    FetchCAs() {

        if [ $# -ne 2 ]; then
            echo "FetchCAs requires two parameters."
            echo "  \$1, 'url', is a url hosting the root CA certificates."
            echo "  \$2, 'cert_dir', is a directory in which to save the certificates."
            exit 1
        fi

        URL="${1}"
        FETCH_CERT_DIR="${2}"
        WGET="/usr/bin/wget"

        # Create a working directory
        if [ -d "${FETCH_CERT_DIR}" ]
        then
            echo "'${FETCH_CERT_DIR}' already exists. Recreating for safety."
            mv "${FETCH_CERT_DIR}" "${FETCH_CERT_DIR}".bak || \
            ( echo "Couldn't move '${FETCH_CERT_DIR}'. Aborting..." && exit 1 )
        fi

        install -d -m 0700 -o root -g root "${FETCH_CERT_DIR}" || \
        ( echo "Could not create '${FETCH_CERT_DIR}'. Aborting..." && exit 1 )

        # Make sure wget is available
        if [ ! -x ${WGET} ]
        then
            echo "The wget utility not found. Attempting to install..."
            yum -y install wget || \
            ( echo "Could not install 'wget', which is required to download the certs. Aborting..." && exit 1 )
        fi

        echo "Attempting to download the root CA certs..."
        ${WGET} -r -l1 -nd -np -A.cer -P "${FETCH_CERT_DIR}" --quiet $URL || \
        ( echo "Could not download certs via 'wget'. Check the url. Quitting..." && \
          exit 1 )
    }

    ######################################
    # Update CA Trust
    ######################################
    UpdateTrust() {

        if [ $# -ne 2]; then
            echo "UpdateTrust requires two parameters."
            echo "  \$1, 'mode', is either '6.0' or '6.5', as determined by the 'GetMode' function."
            echo "  \$2, 'cert_dir', is a directory that contains the root certificates."
            exit 1
        fi

        MODE="${1}"
        UPDATE_CERT_DIR="${2}"

        if [[ "6.5" == "${MODE}" ]]; then
            # Make sure the cert dir exists
            cert_dir="/etc/pki/ca-trust/source/anchors"
            install -d -m 0755 -o root -g root "${cert_dir}"

            echo "Copying certs to $cert_dir..."<commit_msg>Fix typo in `if` test<commit_after>#!/usr/bin/env bash
set -e

#User variables
SOURCEISS3BUCKET="True"
AWSREGION="us-east-1"
AWSCLI_URL="https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"
ROOT_CERT_URL=""
SYSTEMPREPMASTERSCRIPTSOURCE="https://s3.amazonaws.com/systemprep/MasterScripts/systemprep-linuxmaster.py"
SYSTEMPREPPARAMS=( "SaltStates=Highstate"
                   "NoReboot=False"
                   "SourceIsS3Bucket=${SOURCEISS3BUCKET}"
                   "AwsRegion=${AWSREGION}" )

#System variables
SCRIPTNAME=${0}
LOGGER=$(which logger)
TIMESTAMP=$(date -u +"%Y%m%d_%H%M_%S")
WORKINGDIR=/usr/tmp
LOGDIR=/var/log
LOGTAG=systemprep
LOGFILE="${LOGDIR}/${LOGTAG}-${TIMESTAMP}.log"
LOGLINK="${LOGDIR}/${LOGTAG}.log"

# Validate log directory exists
if [[ ! -d ${LOGDIR} ]]; then
  echo "Creating ${LOGDIR} directory." >(${LOGGER} -t "${LOGTAG}" -s 2> /dev/console) 2>&1
  mkdir ${LOGDIR} >(${LOGGER} -i -t "${LOGTAG}" -s 2> /dev/console) 2>&1
fi

# Validate working directory exists
if [[ ! -d ${WORKINGDIR} ]]; then
  echo "Creating ${WORKINGDIR} directory" >(${LOGGER} -t "${LOGTAG}" -s 2> /dev/console) 2>&1
  mkdir ${WORKINGDIR} >(${LOGGER} -i -t "${LOGTAG}" -s 2> /dev/console) 2>&1
fi

# Establish logging to write to the logfile, syslog, and the console
exec > >(tee "${LOGFILE}" | "${LOGGER}" -i -t "${LOGTAG}" -s 2> /dev/console) 2>&1

# Create the link to the logfile
touch ${LOGFILE}
ln -s -f ${LOGFILE} ${LOGLINK}

# Change to the working directory
cd ${WORKINGDIR}

# Create the log file and write out the parameters
echo "Entering SystemPrep script -- ${SCRIPTNAME}"
echo "Writing SystemPrep Parameters to log file..."
for param in "${SYSTEMPREPPARAMS[@]}"; do echo "   ${param}" ; done

# Install root certs, if the root cert url is provided
if [[ -n "${ROOT_CERT_URL}" ]]; then

    #######################################################
    # Are we EL-compatible and do we have 6.5+ behaviour?
    #######################################################
    GetMode() {
        UPDATETRUST="/usr/bin/update-ca-trust"
        CERTUTIL="/usr/bin/certutil"
        if [ -x ${UPDATETRUST} ]
        then
            echo "6.5"
        elif [ -x ${CERTUTIL} ]
        then
            echo "6.0"
        else
            echo "Cannot determine CA update-method. Aborting."
            exit 1
        fi
    }

    #########################################################
    # Try to fetch all CA .cer files from our root_cert_url
    #########################################################
    FetchCAs() {

        if [ $# -ne 2 ]; then
            echo "FetchCAs requires two parameters."
            echo "  \$1, 'url', is a url hosting the root CA certificates."
            echo "  \$2, 'cert_dir', is a directory in which to save the certificates."
            exit 1
        fi

        URL="${1}"
        FETCH_CERT_DIR="${2}"
        WGET="/usr/bin/wget"

        # Create a working directory
        if [ -d "${FETCH_CERT_DIR}" ]
        then
            echo "'${FETCH_CERT_DIR}' already exists. Recreating for safety."
            mv "${FETCH_CERT_DIR}" "${FETCH_CERT_DIR}".bak || \
            ( echo "Couldn't move '${FETCH_CERT_DIR}'. Aborting..." && exit 1 )
        fi

        install -d -m 0700 -o root -g root "${FETCH_CERT_DIR}" || \
        ( echo "Could not create '${FETCH_CERT_DIR}'. Aborting..." && exit 1 )

        # Make sure wget is available
        if [ ! -x ${WGET} ]
        then
            echo "The wget utility not found. Attempting to install..."
            yum -y install wget || \
            ( echo "Could not install 'wget', which is required to download the certs. Aborting..." && exit 1 )
        fi

        echo "Attempting to download the root CA certs..."
        ${WGET} -r -l1 -nd -np -A.cer -P "${FETCH_CERT_DIR}" --quiet $URL || \
        ( echo "Could not download certs via 'wget'. Check the url. Quitting..." && \
          exit 1 )
    }

    ######################################
    # Update CA Trust
    ######################################
    UpdateTrust() {

        if [ $# -ne 2 ]; then
            echo "UpdateTrust requires two parameters."
            echo "  \$1, 'mode', is either '6.0' or '6.5', as determined by the 'GetMode' function."
            echo "  \$2, 'cert_dir', is a directory that contains the root certificates."
            exit 1
        fi

        MODE="${1}"
        UPDATE_CERT_DIR="${2}"

        if [[ "6.5" == "${MODE}" ]]; then
            # Make sure the cert dir exists
            cert_dir="/etc/pki/ca-trust/source/anchors"
            install -d -m 0755 -o root -g root "${cert_dir}"

            echo "Copying certs to $cert_dir..."
{'fraction_non_alphanumeric': 0.20611260053619304, 'fraction_numerical': 0.012868632707774798, 'mean_word_length': 2.4413284132841326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '970230', 'n_tokens_mistral': 3100, 'n_tokens_neox': 2943, 'n_words': 879}