<commit_before>%% -----------------------------------------------------------------------------
%%
%% sqlparse_test.erl: SQL - test driver.
%%
%% Copyright (c) 2012-17 K2 Informatics GmbH.  All Rights Reserved.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% -----------------------------------------------------------------------------

-module(sqlparse_test).

-define(ENV_VAR_FILE_TYPE, ".tst").
-define(ENV_VAR_FILE_WILDCARD, "SOURCEFILES").
-define(ENV_VAR_LOGGING_LEVEL, "LOG").
-define(PARSER_MODULE, sqlparse).
-define(TIMEOUT, 60).
-define(WHITESPACE, " \n\r").

-export([common_test_source/1]).
-export([eunit_test_source/3]).

-define(NODEBUG, true).
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").

%%------------------------------------------------------------------------------
%% Common Test Driver.
%%------------------------------------------------------------------------------

common_test_source(Source) ->
    % ct:pal(info, ?MAX_IMPORTANCE, ?MODULE_STRING ++ ":common_test_source ===>~nSource = ~p~n", [Source]),
    %% -------------------------------------------------------------------------
    %% 1. Source ==> ParseTree
    %% -------------------------------------------------------------------------
    case ?PARSER_MODULE:parsetree_with_tokens(Source) of
        {ok, {ParseTree, _Tokens}} ->
            %% -----------------------------------------------------------------
            %% Test TopDown
            %% -----------------------------------------------------------------
            %% 2. ParseTree ==> Source (=NSource_TD)
            %% -----------------------------------------------------------------
            NSource_TD = case ?PARSER_MODULE:pt_to_string(ParseTree) of
                             {error, Error_TD} ->
                                 ct:pal("~n[TD] Error ParseTree -> NewSource : Source   ~n > ~p", [Source]),
                                 ct:pal("~n[TD] Error ParseTree -> NewSource : ParseTree~n > ~p", [ParseTree]),
                                 throw({error, Error_TD});
                             NS_TD ->
                                 NS_TD
                         end,
            %% -----------------------------------------------------------------
            %% 3. Source (=NSource_TD) ==> ParseTree (=NPT_TD)
            %% -----------------------------------------------------------------
            {ok, {NPTree_TD, _NToks_TD}}
                = try
                      {ok, {NPT_TD, NT_TD}} = ?PARSER_MODULE:parsetree_with_tokens(NSource_TD),
                      {ok, {NPT_TD, NT_TD}}
                  catch
                      ExceptionTD:ReasonTD ->
                          ct:pal("~n[TD] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : Source   ~n > ~p", [Source]),
                          ct:pal("~n[TD] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : NewSource~n > ~p", [NSource_TD]),
                          throw({error, "[TD] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens " ++ atom_to_list(ExceptionTD) ++ " : " ++ ReasonTD})
                  end,
            if ParseTree /= NPTree_TD ->
                ct:pal("~n[TD] Error ParseTree = NPTree : Source      ~n > ~p", [Source]),
                ct:pal("~n[TD] Error ParseTree = NPTree : NewParseTree~n > ~p", [NPTree_TD]),
                ct:pal("~n[TD] Error ParseTree = NPTree : Tokens      ~n > ~p", [_Tokens]),
                ct:pal("~n[TD] Error ParseTree = NPTree : NewTokens   ~n > ~p", [_NToks_TD]);
                true -> ok
            end,
            ?assertEqual(ParseTree, NPTree_TD),
            StringNSource_TD = binary:bin_to_list(NSource_TD),
            StringNSource_TDMultipleSpace = string:str(StringNSource_TD, "  "),
            case StringNSource_TDMultipleSpace of
                0 -> ok;
                _ ->
                    ct:pal("~n[TD] Error redundant whitespace(s) : Source         ~n > ~p", [Source]),
                    ct:pal("~n[TD] Error redundant whitespace(s) : NewSource      ~n > ~p", [StringNSource_TD]),
                    ct:pal("~n[TD] Error redundant whitespace(s) : 1. Redundant WS~n > ~p", [StringNSource_TDMultipleSpace]),
                    throw({error, "[TD] Error redundant whitespace(s)"})
            end,
            %% -----------------------------------------------------------------
            %% Test BottomUp
            %% -----------------------------------------------------------------
            %% 2. ParseTree ==> Source (=NSource_BU)
            %% -----------------------------------------------------------------
            NSource_BU = case ?PARSER_MODULE:pt_to_string_bu(ParseTree) of
                             {error, Error_BU} ->
                                 ct:pal("~n[BU] Error ParseTree -> NewSource : Source   ~n > ~p", [Source]),
                                 ct:pal("~n[BU] Error ParseTree -> NewSource : ParseTree~n > ~p", [ParseTree]),
                                 throw({error, "[BU] Error ParseTree -> NewSource : " ++ Error_BU});
                             NS_BU ->
                                 NS_BU
                         end,
            %% -----------------------------------------------------------------
            %% 3. Source (=NSource_BU) ==> ParseTree (=NPT_BU)
            %% -----------------------------------------------------------------
            {ok, {NPTree_BU, _NToks_BU}}
                = try
                      {ok, {NPT_BU, NT_BU}} = ?PARSER_MODULE:parsetree_with_tokens(NSource_BU),
                      {ok, {NPT_BU, NT_BU}}
                  catch
                      ExceptionBU:ReasonBU ->
                          ct:pal("~n[BU] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : Source   ~n > ~p", [Source]),
                          ct:pal("~n[BU] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : NewSource~n > ~p", [NSource_BU]),
                          throw({error, "[BU] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens " ++ atom_to_list(ExceptionBU) ++ " : " ++ ReasonBU})
                  end,
            if ParseTree /= NPTree_BU ->
                ct:pal("~n[BU] Error ParseTree = NPTree : Source      ~n > ~p", [Source]),
                ct:pal("~n[BU] Error ParseTree = NPTree : NewParseTree~n > ~p", [NPTree_BU]),
                ct:pal("~n[BU] Error ParseTree = NPTree : Tokens      ~n > ~p", [_Tokens]),
                ct:pal("~n[BU] Error ParseTree = NPTree : NewTokens   ~n > ~p", [_NToks_BU]);
                true -> ok
            end,
            ?assertEqual(ParseTree, NPTree_BU),
            StringNSource_BU = binary:bin_to_list(NSource_BU),
            StringNSource_BUMultipleSpace = string:str(StringNSource_BU, "  "),
            case StringNSource_BUMultipleSpace of
                0 -> ok;
                _ ->
                    ct:pal("~n[BU] Error redundant whitespace(s) : Source         ~n > ~p", [Source]),
                    ct:pal("~n[BU] Error redundant whitespace(s) : NewSource      ~n > ~p", [StringNSource_BU]),
                    ct:pal("~n[BU] Error redundant whitespace(s) : 1. Redundant WS~n > ~p", [StringNSource_BUMultipleSpace]),
                    throw({error, "[BU] Error redundant whitespace(s)"})
            end;
        {lex_error, _Error} ->
            ct:pal("~nFailed lex_error : Source~n > ~p", [Source]),
            ct:pal("~nFailed lex_error : Error ~n > ~p", [_Error]),
            throw({error, "Failed lex_error"});
        {parse_error, {_Error, _Tokens}} ->
            ct:pal("~nFailed parse_error : Source~n > ~p", [Source]),
            ct:pal("~nFailed parse_error : Tokens~n > ~p", [_Tokens]),
            ct:pal("~nFailed parse_error : Error ~n > ~p", [_Error]),
            throw({error, "Failed parse_error"})
    end.

%%------------------------------------------------------------------------------
%% EUnit Test Driver - Determine Files.
%%------------------------------------------------------------------------------

eunit_test_() ->
    WCard = case os:getenv(?ENV_VAR_FILE_WILDCARD) of
                SourceFiles when is_list(SourceFiles) ->
                    SourceFiles;
                _ ->
                    "*"
            end ++ ?ENV_VAR_FILE_TYPE,
    Logs = case os:getenv(?ENV_VAR_LOGGING_LEVEL) of
               V when length(V) > 0 ->
                   case lists:map(
                       fun(VStr) ->
                           list_to_integer(VStr)
                       end, re:split(V, ",", [{return, list}])) of
                       VIs when is_list(VIs) -> VIs;
                       _ -> [0]
                   end;
               _ -> [0]
           end,
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nFile = ~s, Logs = ~p~n", [WCard, Logs]),
    {ok, Cwd} = file:get_cwd(),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nCwd: ~p~n", [Cwd]),
    RootPath = lists:reverse(filename:split(Cwd)),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nRootPath: ~p~n", [RootPath]),
    TestDir = filename:join(lists:reverse(["test" | RootPath])),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nTestDir: ~p~n", [TestDir]),
    TestFiles = lists:sort([filename:join(TestDir, T)
        || T <- filelib:wildcard(WCard, TestDir)]),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nTestFiles: ~p~n", [TestFiles]),
    group_gen(TestFiles, Logs).

%%------------------------------------------------------------------------------
%% EUnit Test Driver - Processing Test Cases.
%%------------------------------------------------------------------------------

-define(D(__Lvl, __Fmt, __Args),
    case lists:member(__Lvl, Logs) of
        true -> io:format(standard_io, __Fmt, __Args);
        _ -> ok
    end).
-define(D(__Lvl, __Msg),
    case lists:member(__Lvl, Logs) of
        true -> io:format(standard_io, __Msg, []);
        _ -> ok
    end).<commit_msg>Cleaning up the 'test' directory.<commit_after>%% -----------------------------------------------------------------------------
%%
%% sqlparse_test.erl: SQL - test driver.
%%
%% Copyright (c) 2012-17 K2 Informatics GmbH.  All Rights Reserved.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% -----------------------------------------------------------------------------

-module(sqlparse_test).

-define(ENV_VAR_FILE_TYPE, ".tst").
-define(ENV_VAR_FILE_WILDCARD, "SOURCEFILES").
-define(ENV_VAR_LOGGING_LEVEL, "LOG").
-define(PARSER_MODULE, sqlparse).
-define(TIMEOUT, 60).
-define(WHITESPACE, " \n\r").

-export([common_test_source/1]).
-export([eunit_test_source/3]).

-define(NODEBUG, true).
-include_lib("common_test/include/ct.hrl").
-include_lib("eunit/include/eunit.hrl").

%%------------------------------------------------------------------------------
%% Common Test Driver.
%%------------------------------------------------------------------------------

common_test_source(Source) ->
    % ct:pal(info, ?MAX_IMPORTANCE, ?MODULE_STRING ++ ":common_test_source ===>~nSource = ~p~n", [Source]),
    %% -------------------------------------------------------------------------
    %% 1. Source ==> ParseTree
    %% -------------------------------------------------------------------------
    case ?PARSER_MODULE:parsetree_with_tokens(Source) of
        {ok, {ParseTree, _Tokens}} ->
            %% -----------------------------------------------------------------
            %% Test TopDown
            %% -----------------------------------------------------------------
            %% 2. ParseTree ==> Source (=NSource_TD)
            %% -----------------------------------------------------------------
            NSource_TD = case ?PARSER_MODULE:pt_to_string(ParseTree) of
                             {error, Error_TD} ->
                                 ct:pal("~n[TD] Error ParseTree -> NewSource : Source   ~n > ~p", [Source]),
                                 ct:pal("~n[TD] Error ParseTree -> NewSource : ParseTree~n > ~p", [ParseTree]),
                                 throw({error, Error_TD});
                             NS_TD ->
                                 NS_TD
                         end,
            %% -----------------------------------------------------------------
            %% 3. Source (=NSource_TD) ==> ParseTree (=NPT_TD)
            %% -----------------------------------------------------------------
            {ok, {NPTree_TD, _NToks_TD}}
                = try
                      {ok, {NPT_TD, NT_TD}} = ?PARSER_MODULE:parsetree_with_tokens(NSource_TD),
                      {ok, {NPT_TD, NT_TD}}
                  catch
                      ExceptionTD:ReasonTD ->
                          ct:pal("~n[TD] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : Source   ~n > ~p", [Source]),
                          ct:pal("~n[TD] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : NewSource~n > ~p", [NSource_TD]),
                          throw({error, "[TD] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens " ++ atom_to_list(ExceptionTD) ++ " : " ++ ReasonTD})
                  end,
            if ParseTree /= NPTree_TD ->
                ct:pal("~n[TD] Error ParseTree = NPTree : Source      ~n > ~p", [Source]),
                ct:pal("~n[TD] Error ParseTree = NPTree : NewParseTree~n > ~p", [NPTree_TD]),
                ct:pal("~n[TD] Error ParseTree = NPTree : Tokens      ~n > ~p", [_Tokens]),
                ct:pal("~n[TD] Error ParseTree = NPTree : NewTokens   ~n > ~p", [_NToks_TD]);
                true -> ok
            end,
            ?assertEqual(ParseTree, NPTree_TD),
            StringNSource_TD = binary:bin_to_list(NSource_TD),
            StringNSource_TDMultipleSpace = string:str(StringNSource_TD, "  "),
            case StringNSource_TDMultipleSpace of
                0 -> ok;
                _ ->
                    ct:pal("~n[TD] Error redundant whitespace(s) : Source         ~n > ~p", [Source]),
                    ct:pal("~n[TD] Error redundant whitespace(s) : NewSource      ~n > ~p", [StringNSource_TD]),
                    ct:pal("~n[TD] Error redundant whitespace(s) : 1. Redundant WS~n > ~p", [StringNSource_TDMultipleSpace]),
                    throw({error, "[TD] Error redundant whitespace(s)"})
            end,
            %% -----------------------------------------------------------------
            %% Test BottomUp
            %% -----------------------------------------------------------------
            %% 2. ParseTree ==> Source (=NSource_BU)
            %% -----------------------------------------------------------------
            NSource_BU = case ?PARSER_MODULE:pt_to_string_bu(ParseTree) of
                             {error, Error_BU} ->
                                 ct:pal("~n[BU] Error ParseTree -> NewSource : Source   ~n > ~p", [Source]),
                                 ct:pal("~n[BU] Error ParseTree -> NewSource : ParseTree~n > ~p", [ParseTree]),
                                 throw({error, "[BU] Error ParseTree -> NewSource : " ++ Error_BU});
                             NS_BU ->
                                 NS_BU
                         end,
            %% -----------------------------------------------------------------
            %% 3. Source (=NSource_BU) ==> ParseTree (=NPT_BU)
            %% -----------------------------------------------------------------
            {ok, {NPTree_BU, _NToks_BU}}
                = try
                      {ok, {NPT_BU, NT_BU}} = ?PARSER_MODULE:parsetree_with_tokens(NSource_BU),
                      {ok, {NPT_BU, NT_BU}}
                  catch
                      ExceptionBU:ReasonBU ->
                          ct:pal("~n[BU] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : Source   ~n > ~p", [Source]),
                          ct:pal("~n[BU] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens : NewSource~n > ~p", [NSource_BU]),
                          throw({error, "[BU] Error " ++ ?MODULE_STRING ++ ":parsetree_with_tokens " ++ atom_to_list(ExceptionBU) ++ " : " ++ ReasonBU})
                  end,
            if ParseTree /= NPTree_BU ->
                ct:pal("~n[BU] Error ParseTree = NPTree : Source      ~n > ~p", [Source]),
                ct:pal("~n[BU] Error ParseTree = NPTree : NewParseTree~n > ~p", [NPTree_BU]),
                ct:pal("~n[BU] Error ParseTree = NPTree : Tokens      ~n > ~p", [_Tokens]),
                ct:pal("~n[BU] Error ParseTree = NPTree : NewTokens   ~n > ~p", [_NToks_BU]);
                true -> ok
            end,
            ?assertEqual(ParseTree, NPTree_BU),
            StringNSource_BU = binary:bin_to_list(NSource_BU),
            StringNSource_BUMultipleSpace = string:str(StringNSource_BU, "  "),
            case StringNSource_BUMultipleSpace of
                0 -> ok;
                _ ->
                    ct:pal("~n[BU] Error redundant whitespace(s) : Source         ~n > ~p", [Source]),
                    ct:pal("~n[BU] Error redundant whitespace(s) : NewSource      ~n > ~p", [StringNSource_BU]),
                    ct:pal("~n[BU] Error redundant whitespace(s) : 1. Redundant WS~n > ~p", [StringNSource_BUMultipleSpace]),
                    throw({error, "[BU] Error redundant whitespace(s)"})
            end;
        {lex_error, _Error} ->
            ct:pal("~nFailed lex_error : Source~n > ~p", [Source]),
            ct:pal("~nFailed lex_error : Error ~n > ~p", [_Error]),
            throw({error, "Failed lex_error"});
        {parse_error, {_Error, _Tokens}} ->
            ct:pal("~nFailed parse_error : Source~n > ~p", [Source]),
            ct:pal("~nFailed parse_error : Tokens~n > ~p", [_Tokens]),
            ct:pal("~nFailed parse_error : Error ~n > ~p", [_Error]),
            throw({error, "Failed parse_error"})
    end.

%%------------------------------------------------------------------------------
%% EUnit Test Driver - Determine Files.
%%------------------------------------------------------------------------------

eunit_test_() ->
    WCard = case os:getenv(?ENV_VAR_FILE_WILDCARD) of
                SourceFiles when is_list(SourceFiles) ->
                    SourceFiles;
                _ ->
                    "*"
            end ++ ?ENV_VAR_FILE_TYPE,
    Logs = case os:getenv(?ENV_VAR_LOGGING_LEVEL) of
               V when length(V) > 0 ->
                   case lists:map(
                       fun(VStr) ->
                           list_to_integer(VStr)
                       end, re:split(V, ",", [{return, list}])) of
                       VIs when is_list(VIs) -> VIs;
                       _ -> [0]
                   end;
               _ -> [0]
           end,
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nFile = ~s, Logs = ~p~n", [WCard, Logs]),
    {ok, Cwd} = file:get_cwd(),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nCwd: ~p~n", [Cwd]),
    RootPath = lists:reverse(filename:split(Cwd)),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nRootPath: ~p~n", [RootPath]),
    TestDir1 = filename:join(lists:reverse(["test" | RootPath])),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nTestDir1: ~p~n", [TestDir1]),
    TestDir2 = filename:join(lists:reverse(["eunit", "generated", "test"] ++ RootPath)),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nTestDir2: ~p~n", [TestDir2]),
    TestFiles = lists:sort([filename:join(TestDir1, T) || T <- filelib:wildcard(WCard, TestDir1)]
    ++ [filename:join(TestDir2, T) || T <- filelib:wildcard(WCard, TestDir2)]
    ),
    ?debugFmt(?MODULE_STRING ++ ":eunit_test_ ===>~nTestFiles: ~p~n", [TestFiles]),
    group_gen(TestFiles, Logs).

%%------------------------------------------------------------------------------
%% EUnit Test Driver - Processing Test Cases.
%%------------------------------------------------------------------------------

-define(D(__Lvl, __Fmt, __Args),
    case lists:member(__Lvl, Logs) of
        true -> io:format(standard_io, __Fmt, __Args);
        _ -> ok
    end).
-define(D(__Lvl, __Msg),
    case lists:member(__Lvl, Logs) of
        true -> io:format(standard_io, __Msg, []);
        _ -> ok
    end).
{'fraction_non_alphanumeric': 0.26336942645042316, 'fraction_numerical': 0.003026147808407017, 'mean_word_length': 1.2991629524948365, 'pattern_counts': {'":': 25, '<': 6, '<?xml version=': 0, '>': 152, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3270650', 'n_tokens_mistral': 6011, 'n_tokens_neox': 5739, 'n_words': 1214}