<commit_before>#!/usr/bin/env sh
# Unit tests for the octokit.sh script.

SCRIPT="octokit.sh"
JQ="${OCTOKIT_SH_JQ_BIN:-jq}"
JQ_V="$(jq --version 2>&1 | awk '{ print $3 }')"

_main() {
    local cmd ret

    cmd="$1" && shift
    "$cmd" "$@"
    ret=$?

    [ $ret -eq 0 ] || printf 'Fail: %s\n' "$cmd" 1>&2
    exit $ret
}

test_format_json() {
    # Test output without filtering through jq.

    local output
    local is_fail=0

    $SCRIPT -j _format_json foo=Foo bar=123 baz=true qux=Qux=Qux quux='Multi-line
string' | {
        read -r output

        printf '%s\n' "$output" | grep -q -E '^{' || {
            printf 'JSON does not start with a { char.\n'; is_fail=1 ;}
        printf '%s\n' "$output" | grep -q -E '}$' || {
            printf 'JSON does not end with a } char.\n'; is_fail=1 ;}
        printf '%s\n' "$output" | grep -q -E '"foo": "Foo"' || {
            printf 'JSON does not contain "foo": "Foo" text.\n'; is_fail=1 ;}
        printf '%s\n' "$output" | grep -q -E '"Multi-line\\nstring"' || {
            printf 'JSON does not have properly formatted multiline string.\n'; is_fail=1 ;}

        if [ "$is_fail" -ne 1 ] ; then
            return 0
        else
            printf 'Unexpected JSON output: `%s`\n' "$output"
            return 1
        fi
    }
}

test_format_json_jq() {
    # Test output after filtering through jq.

    local output keys vals expected_out

    $SCRIPT _format_json foo=Foo bar=123 baz=true qux=Qux=Qux quux='Multi-line
string' | {
        read -r output

        keys=$(printf '%s\n' "$output" | jq -r -c 'keys | .[]' | sort | paste -s -d',' -)
        vals=$(printf '%s\n' "$output" | jq -r -c '.[]' | sort | paste -s -d',' -)

        if [ 'bar,baz,foo,quux,qux' = "$keys" ] && [ '123,Foo,Multi-line,Qux=Qux,string,true' = "$vals" ] ; then
            return 0
        else
            printf 'Expected output does not match output: `%s` != `%s`\n' \
                "$expected_out" "$output"
            return 1
        fi
    }<commit_msg>Add unit test for _format_urlencode<commit_after>#!/usr/bin/env sh
# Unit tests for the octokit.sh script.

SCRIPT="octokit.sh"
JQ="${OCTOKIT_SH_JQ_BIN:-jq}"
JQ_V="$(jq --version 2>&1 | awk '{ print $3 }')"

_main() {
    local cmd ret

    cmd="$1" && shift
    "$cmd" "$@"
    ret=$?

    [ $ret -eq 0 ] || printf 'Fail: %s\n' "$cmd" 1>&2
    exit $ret
}

test_format_json() {
    # Test output without filtering through jq.

    local output
    local is_fail=0

    $SCRIPT -j _format_json foo=Foo bar=123 baz=true qux=Qux=Qux quux='Multi-line
string' | {
        read -r output

        printf '%s\n' "$output" | grep -q -E '^{' || {
            printf 'JSON does not start with a { char.\n'; is_fail=1 ;}
        printf '%s\n' "$output" | grep -q -E '}$' || {
            printf 'JSON does not end with a } char.\n'; is_fail=1 ;}
        printf '%s\n' "$output" | grep -q -E '"foo": "Foo"' || {
            printf 'JSON does not contain "foo": "Foo" text.\n'; is_fail=1 ;}
        printf '%s\n' "$output" | grep -q -E '"Multi-line\\nstring"' || {
            printf 'JSON does not have properly formatted multiline string.\n'; is_fail=1 ;}

        if [ "$is_fail" -ne 1 ] ; then
            return 0
        else
            printf 'Unexpected JSON output: `%s`\n' "$output"
            return 1
        fi
    }
}

test_format_urlencode() {
    # _format_urlencode 

    local output
    local expected_out='foo=Foo%20Foo&bar=%3CBar%3E%26%2FBar%2F'

    $SCRIPT _format_urlencode foo='Foo Foo' bar='<Bar>&/Bar/' | {
        read -r output

        if [ "$expected_out" = "$output" ]; then
            return 0
        else
            printf 'Expected output does not match output: `%s` != `%s`\n' \
                "$expected_out" "$output"
            return 1
        fi
    }
}

test_format_json_jq() {
    # Test output after filtering through jq.

    local output keys vals expected_out

    $SCRIPT _format_json foo=Foo bar=123 baz=true qux=Qux=Qux quux='Multi-line
string' | {
        read -r output

        keys=$(printf '%s\n' "$output" | jq -r -c 'keys | .[]' | sort | paste -s -d',' -)
        vals=$(printf '%s\n' "$output" | jq -r -c '.[]' | sort | paste -s -d',' -)

        if [ 'bar,baz,foo,quux,qux' = "$keys" ] && [ '123,Foo,Multi-line,Qux=Qux,string,true' = "$vals" ] ; then
            return 0
        else
            printf 'Expected output does not match output: `%s` != `%s`\n' \
                "$expected_out" "$output"
            return 1
        fi
    }
{'fraction_non_alphanumeric': 0.16977777777777778, 'fraction_numerical': 0.013777777777777778, 'mean_word_length': 2.0639891082368957, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '565769', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1579, 'n_words': 449}