<commit_before>#!/usr/bin/env bats
# Tests for `deepdive sql eval` formats

. "$BATS_TEST_DIRNAME"/env.sh >&2
PATH="$DEEPDIVE_SOURCE_ROOT/util/test:$PATH"

@test "$DBVARIANT deepdive sql eval format=json works" {
    q="SELECT 123::bigint as i
            , 45.678 as float
            , TRUE as t
            , FALSE as f
            , 'foo bar baz'::text AS s
            , NULL::text AS n
            , ARRAY[1,2,3] AS num_arr
            , ARRAY[1.2,3.45,67.890] AS float_arr
            , ARRAY[ 'easy'
                   , '123'
                   , 'abc'
                   , 'two words'
                   ] as text_arr
            , ARRAY[ '.'
                   , ','
                   , '.'
                   , '{'
                   , '}'
                   , '['
                   , ']'
                   , '('
                   , ')'
                   , '\"'
                   , '\\'
                   ] AS punctuations
            , ARRAY[ 'asdf  qwer"$'\t'"zxcv"$'\n'"1234'
                   , '\"I''m your father,\" said <NAME>ader.'
                   , '"'{"csv in a json": "a,b c,\",\",\"line '\'\''1'\'\'$'\n''line \"\"2\"\"",  "foo":123,'$'\n''"bar":45.678}'"'
                   ] AS torture_arr
    "
    expected='
        {
          "i": 123,
          "float": 45.678,
          "t": true,
          "f": false,
          "s": "foo bar baz",
          "n": null,
          "num_arr": [
            1,
            2,
            3
          ],
          "float_arr": [
            1.2,
            3.45,
            67.89
          ],
          "text_arr": [
            "easy",
            "123",
            "abc",
            "two words"
          ],
          "punctuations": [
            ".",
            ",",
            ".",
            "{",
            "}",
            "[",
            "]",
            "(",
            ")",
            "\"",
            "\\"
          ],
          "torture_arr": [
            "asdf  qwer\tzxcv\n1234",
            "\"I'\''m your father,\" said <NAME>.",
            "{\"csv in a json\": \"a,b c,\\\",\\\",\\\"line '\''1'\''\nline \\\"\\\"2\\\"\\\"\",  \"foo\":123,\n\"bar\":45.678}"
          ]
        }
    '
    actual=$(deepdive sql eval "$q" format=json)
    # test whether two JSONs are the same
    compare_json "$expected" "$actual"
}
<commit_msg>Fixes deepdive sql eval test input SQL for Greenplum<commit_after>#!/usr/bin/env bats
# Tests for `deepdive sql eval` formats

. "$BATS_TEST_DIRNAME"/env.sh >&2
PATH="$DEEPDIVE_SOURCE_ROOT/util/test:$PATH"

@test "$DBVARIANT deepdive sql eval format=json works" {
    q="SELECT 123::bigint as i
            , 45.678 as float
            , TRUE as t
            , FALSE as f
            , 'foo bar baz'::text AS s
            , NULL::text AS n
            , ARRAY[1,2,3] AS num_arr
            , ARRAY[1.2,3.45,67.890] AS float_arr
            , ARRAY[ 'easy'
                   , '123'
                   , 'abc'
                   , 'two words'
                   ] as text_arr
            , ARRAY[ '.'
                   , ','
                   , '.'
                   , '{'
                   , '}'
                   , '['
                   , ']'
                   , '('
                   , ')'
                   , '\"'
                   , E'\\\\' -- XXX Greenplum doesn't like the simpler '\\'
                   ] AS punctuations
            , ARRAY[ 'asdf  qwer"$'\t'"zxcv"$'\n'"1234'
                   , '\"I''m your father,\" said <NAME>.'
                   , E'"'{"csv in a json": "a,b c,\\",\\",\\"line '\'\''1'\'\'$'\n''line \\"\\"2\\"\\"",  "foo":123,'$'\n''"bar":45.678}'"'
                     -- XXX Greenplum (or older PostgreSQL 8.x) treats backslashes as escapes in strings '...'
                     -- and E'...' is a consistent way to write backslashes in string literal across versions
                   ] AS torture_arr
    "
    expected='
        {
          "i": 123,
          "float": 45.678,
          "t": true,
          "f": false,
          "s": "foo bar baz",
          "n": null,
          "num_arr": [
            1,
            2,
            3
          ],
          "float_arr": [
            1.2,
            3.45,
            67.89
          ],
          "text_arr": [
            "easy",
            "123",
            "abc",
            "two words"
          ],
          "punctuations": [
            ".",
            ",",
            ".",
            "{",
            "}",
            "[",
            "]",
            "(",
            ")",
            "\"",
            "\\"
          ],
          "torture_arr": [
            "asdf  qwer\tzxcv\n1234",
            "\"I'\''m your father,\" said <NAME>.",
            "{\"csv in a json\": \"a,b c,\\\",\\\",\\\"line '\''1'\''\nline \\\"\\\"2\\\"\\\"\",  \"foo\":123,\n\"bar\":45.678}"
          ]
        }
    '
    actual=$(deepdive sql eval "$q" format=json)
    # test whether two JSONs are the same
    compare_json "$expected" "$actual"
}

{'fraction_non_alphanumeric': 0.18544506816359263, 'fraction_numerical': 0.030673616680032077, 'mean_word_length': 0.9085692425401684, 'pattern_counts': {'":': 34, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4877582', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1549, 'n_words': 315}