<commit_before>#!/usr/bin/env bash
# Verifies environment and launches docker to execute test-runner.sh

# 1. I can pick up a brand new laptop, and as long as I have docker installed, everything will just work if I do ./test.sh <fw>
# 2. I want test.sh to default to running _all_ tests for that framework.
# 3. I want to be able to pass -m or -k to pytest
# 4. If I pass `all` instead of a fw name, it will run all frameworks
# 5. test.sh should validate i have the AWS keys, and a CLUSTER_URL set, but it need not verify the azure keys / security / etc

# Exit immediately on errors
set -e

REPO_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
FRAMEWORK_LIST=$(ls $REPO_ROOT_DIR/frameworks | sort)

function usage()
{
    echo "Usage: $0 [-m MARKEXPR] [-k EXPRESSION] [-p PATH] [-s] all|<framework-name>"
    echo "-m passed to pytest directly [default -m \"sanity and not azure\"]"
    echo "-k passed to pytest directly [default NONE]"
    echo "-p PATH to cluster SSH key [default ~/.ssh/ccm.pem]"
    echo "-s run in strict mode (sets \$SECURITY=\"strict\")"
    echo "Cluster must be created and \$CLUSTER_URL set"
    echo "AWS credentials must exist in the variables:"
    echo "      \$AWS_ACCESS_KEY_ID"
    echo "      \$AWS_SECRET_ACCESS_KEY"
    echo "Azure tests will run if thses variables are set:"
    echo "      \$AWS_ACCESS_KEY_ID"
    echo "      \$AWS_SECRET_ACCESS_KEY"
    echo "      \$AZURE_CLIENT_ID"
    echo "      \$AZURE_CLIENT_SECRET"
    echo "      \$AZURE_TENANT_ID"
    echo "      \$AZURE_STORAGE_ACCOUNT"
    echo "      \$AZURE_STORAGE_KEY"
    echo "  (changes the -m default to \"sanity\")"
    echo "Set \$STUB_UNIVERSE_URL to bypass build"
    echo "  (invalid when building all frameworks)"
    echo "Current frameworks:"
    for framework in $FRAMEWORK_LIST; do
        echo "       $framework"
    done
    exit 1
}

if [ "$#" -eq "0" ]; then
    usage
fi

if [ -z "$CLUSTER_URL" ]; then
    echo "Cluster not found. Create and configure one then set \$CLUSTER_URL."
    exit 1
fi

if [ -z "$AWS_ACCESS_KEY_ID" -o -z "$AWS_SECRET_ACCESS_KEY" ]; then
    echo "AWS credentials not found (\$AWS_ACCESS_KEY_ID and \$AWS_SECRET_ACCESS_KEY)."
    exit 1
fi

security="permissive"
pytest_m="sanity and not azure"
pytest_k=""
azure_args=""
ssh_path="${HOME}/.ssh/ccm.pem"

# If AZURE variables are given, change default -m and prepare args for docker
if [ -n "$AZURE_DEV_CLIENT_ID" -a -n "$AZURE_DEV_CLIENT_SECRET" -a \
        -n "$AZURE_DEV_TENANT_ID" -a -n "$AZURE_DEV_STORAGE_ACCOUNT" -a \
        -n "$AZURE_DEV_STORAGE_KEY" ]; then
azure_args=$(cat<<EOFF
    -e AZURE_CLIENT_ID=$AZURE_DEV_CLIENT_ID \
    -e AZURE_CLIENT_SECRET=$AZURE_DEV_CLIENT_SECRET \
    -e AZURE_TENANT_ID=$AZURE_DEV_TENANT_ID \
    -e AZURE_STORAGE_ACCOUNT=$AZURE_DEV_STORAGE_ACCOUNT \
    -e AZURE_STORAGE_KEY=$AZURE_DEV_STORAGE_KEY
EOFF
)
    pytest_m="sanity"
fi<commit_msg>Read AWS credentials from ~/.aws/credentials if it exists<commit_after>#!/usr/bin/env bash
# Verifies environment and launches docker to execute test-runner.sh

# 1. I can pick up a brand new laptop, and as long as I have docker installed, everything will just work if I do ./test.sh <fw>
# 2. I want test.sh to default to running _all_ tests for that framework.
# 3. I want to be able to pass -m or -k to pytest
# 4. If I pass `all` instead of a fw name, it will run all frameworks
# 5. test.sh should validate i have the AWS keys, and a CLUSTER_URL set, but it need not verify the azure keys / security / etc

# Exit immediately on errors
set -e

REPO_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
FRAMEWORK_LIST=$(ls $REPO_ROOT_DIR/frameworks | sort)

function usage()
{
    echo "Usage: $0 [-m MARKEXPR] [-k EXPRESSION] [-p PATH] [-s] all|<framework-name>"
    echo "-m passed to pytest directly [default -m \"sanity and not azure\"]"
    echo "-k passed to pytest directly [default NONE]"
    echo "-p PATH to cluster SSH key [default ~/.ssh/ccm.pem]"
    echo "-s run in strict mode (sets \$SECURITY=\"strict\")"
    echo "Cluster must be created and \$CLUSTER_URL set"
    echo "AWS credentials must exist in the variables:"
    echo "      \$AWS_ACCESS_KEY_ID"
    echo "      \$AWS_SECRET_ACCESS_KEY"
    echo "Azure tests will run if thses variables are set:"
    echo "      \$AWS_ACCESS_KEY_ID"
    echo "      \$AWS_SECRET_ACCESS_KEY"
    echo "      \$AZURE_CLIENT_ID"
    echo "      \$AZURE_CLIENT_SECRET"
    echo "      \$AZURE_TENANT_ID"
    echo "      \$AZURE_STORAGE_ACCOUNT"
    echo "      \$AZURE_STORAGE_KEY"
    echo "  (changes the -m default to \"sanity\")"
    echo "Set \$STUB_UNIVERSE_URL to bypass build"
    echo "  (invalid when building all frameworks)"
    echo "Current frameworks:"
    for framework in $FRAMEWORK_LIST; do
        echo "       $framework"
    done
    exit 1
}

if [ "$#" -eq "0" ]; then
    usage
fi

if [ -z "$CLUSTER_URL" ]; then
    echo "Cluster not found. Create and configure one then set \$CLUSTER_URL."
    exit 1
fi

if [ -z "$AWS_ACCESS_KEY_ID" -o -z "$AWS_SECRET_ACCESS_KEY" ]; then
    CREDENTIALS_FILE="$HOME/.aws/credentials"
    if  [ -e "$CREDENTIALS_FILE" ]
    then
        echo "Checking $CREDENTIALS_FILE"
        SED_ARGS='s/^.*=\s*//g'
        AWS_ACCESS_KEY_ID=$( grep -oE "^aws_access_key_id\s*=\s*\S+" $CREDENTIALS_FILE | sed $SED_ARGS )
        AWS_SECRET_ACCESS_KEY=$( grep -oE "^aws_secret_access_key\s*=\s*\S+" $CREDENTIALS_FILE | sed $SED_ARGS )
    fi
    if [ -z "$AWS_ACCESS_KEY_ID" -o -z "$AWS_SECRET_ACCESS_KEY" ]; then
        echo "AWS credentials not found (\$AWS_ACCESS_KEY_ID and \$AWS_SECRET_ACCESS_KEY)."
        exit 1
    fi
fi

security="permissive"
pytest_m="sanity and not azure"
pytest_k=""
azure_args=""
ssh_path="${HOME}/.ssh/ccm.pem"

# If AZURE variables are given, change default -m and prepare args for docker
if [ -n "$AZURE_DEV_CLIENT_ID" -a -n "$AZURE_DEV_CLIENT_SECRET" -a \
        -n "$AZURE_DEV_TENANT_ID" -a -n "$AZURE_DEV_STORAGE_ACCOUNT" -a \
        -n "$AZURE_DEV_STORAGE_KEY" ]; then
azure_args=$(cat<<EOFF
    -e AZURE_CLIENT_ID=$AZURE_DEV_CLIENT_ID \
    -e AZURE_CLIENT_SECRET=$AZURE_DEV_CLIENT_SECRET \
    -e AZURE_TENANT_ID=$AZURE_DEV_TENANT_ID \
    -e AZURE_STORAGE_ACCOUNT=$AZURE_DEV_STORAGE_ACCOUNT \
    -e AZURE_STORAGE_KEY=$AZURE_DEV_STORAGE_KEY
EOFF
)
    pytest_m="sanity"
fi
{'fraction_non_alphanumeric': 0.11580940359734933, 'fraction_numerical': 0.0034711265383401703, 'mean_word_length': 3.159448818897638, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1603515', 'n_tokens_mistral': 2439, 'n_tokens_neox': 2322, 'n_words': 704}