<commit_before>#!/bin/bash
EXIT=0
BASE_URL=${1:-https://snippets.mozilla.com}
URLS=(
    "/"
    "/healthz/"
    "/readiness/"
    "/robots.txt"
    "/contribute.json"
    "/5/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/en-US/release/Windows_NT%206.1/default/default/"
    "/json/1/Fennec/55.0.2/20170815231002/Android_arm-eabi-gcc3/en-GB/release/Linux%2022/default/default/"
    "/feeds/snippets-enabled.ics"
)

function check_http_code {
    echo -n "Checking URL ${1} "
    curl -k -L -s -o /dev/null -I -w "%{http_code}" $1 | grep ${2:-200} > /dev/null
    if [ $? -eq 0 ];
    then
        echo "OK"
    else
        echo "Failed"
        EXIT=1
    fi
}

function check_zero_content_length {
    echo -n "Checking zero content length of URL ${1} "
    test=$(curl -L -s ${1} | wc -c);
    if [[ $test -eq 0 ]];
    then
        echo "OK"
    else
        echo "Failed"
        EXIT=1
    fi
}

for url in ${URLS[*]}
do
    check_http_code ${BASE_URL}${url}
done

# Check a page that throws 404. Not ideal but will surface 500s
check_http_code ${BASE_URL}/foo 404

check_zero_content_length ${BASE_URL}/4/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/xx/release/Windows_NT%206.1/default/default/
check_zero_content_length ${BASE_URL}/5/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/xx/release/Windows_NT%206.1/default/default/

exit ${EXIT}<commit_msg>Add /6/ urls in acceptance tests.<commit_after>#!/bin/bash
EXIT=0
BASE_URL=${1:-https://snippets.mozilla.com}
URLS=(
    "/"
    "/healthz/"
    "/readiness/"
    "/robots.txt"
    "/contribute.json"
    "/5/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/en-US-test/release/Windows_NT%206.1/default/default/"
    "/6/Firefox/62.0.1/20160922113459/WINNT_x86-msvc/en-US-test/release/Windows_NT%206.1/default/default/"
    "/json/1/Fennec/55.0.2/20170815231002/Android_arm-eabi-gcc3/en-GB/release/Linux%2022/default/default/"
    "/feeds/snippets-enabled.ics"
)

function check_http_code {
    echo -n "Checking URL ${1} "
    curl -k -L -s -o /dev/null -I -w "%{http_code}" $1 | grep ${2:-200} > /dev/null
    if [ $? -eq 0 ];
    then
        echo "OK"
    else
        echo "Failed"
        EXIT=1
    fi
}

function check_zero_content_length {
    echo -n "Checking zero content length of URL ${1} "
    test=$(curl -L -s ${1} | wc -c);
    if [[ $test -eq 0 ]];
    then
        echo "OK"
    else
        echo "Failed"
        EXIT=1
    fi
}

for url in ${URLS[*]}
do
    check_http_code ${BASE_URL}${url}
done

# Check a page that throws 404. Not ideal but will surface 500s
check_http_code ${BASE_URL}/foo 404

check_zero_content_length ${BASE_URL}/4/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/xx/release/Windows_NT%206.1/default/default/
check_zero_content_length ${BASE_URL}/5/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/xx/release/Windows_NT%206.1/default/default/
check_zero_content_length ${BASE_URL}/6/Firefox/56.0.1/20160922113459/WINNT_x86-msvc/xx/release/Windows_NT%206.1/default/default/

exit ${EXIT}
{'fraction_non_alphanumeric': 0.1617350369872226, 'fraction_numerical': 0.09919300605245461, 'mean_word_length': 2.8990825688073394, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3589488', 'n_tokens_mistral': 1371, 'n_tokens_neox': 1207, 'n_words': 207}