<commit_before>-module(dderl_sqlbox).

-export([ pretty_from_pt/1
        , boxed_from_pt/1
        , flat_from_box/1
        , pretty_from_box/1
        , box_to_json/1
        , foldb/3
        ]).

-include("dderl.hrl").
-include("dderl_sqlbox.hrl").

-define(DefCollInd,10).  % First indentation level which will be collapsed by default

-spec validate_children([#box{}]) -> ok | {error, binary()}.
validate_children([]) -> ok;
validate_children([#box{}]) -> ok;
validate_children(Children) ->
    case length(lists:usort([C#box.ind || C <- Children])) of
        1 -> ok;
        _ -> {error, <<"Mixed identation error">>}
    end.

-spec flat_from_box([#box{}] | #box{}) -> binary().
flat_from_box([]) -> <<>>;
flat_from_box(#box{name= <<",">> ,children=[]}) -> <<", ">>;
flat_from_box(#box{name=Name,children=[]}) ->
    iolist_to_binary([binary_to_list(Name), " "]);
flat_from_box(#box{name= <<>>,children=CH}) ->
    iolist_to_binary([[flat_from_box(C) || C <-CH]]);
flat_from_box(#box{name=Name,children=CH}) ->
    iolist_to_binary([Name," ",[flat_from_box(C) || C <-CH]]);
flat_from_box([Box|Boxes]) ->
    iolist_to_binary([flat_from_box(Box),flat_from_box(Boxes)]).

-spec pretty_from_pt(term()) -> binary() | {error, binary()}.
pretty_from_pt(ParseTree) ->
    case foldb(ParseTree) of
        {error, Reason} -> {error, Reason};
        SqlBox -> pretty_from_box(SqlBox)
    end.

-spec pretty_from_box([#box{}] | #box{}) -> binary().
pretty_from_box([]) -> [];
pretty_from_box(#box{collapsed=true}=Box) -> flat_from_box(Box);
pretty_from_box(#box{name= <<>>,children=[]}) -> <<>>;
pretty_from_box(#box{ind=Ind,name=Name,children=[]}) ->
    iolist_to_binary([indent(Ind), Name, "\r\n"]);
pretty_from_box(#box{ind=Ind,name= <<>>,children=CH}) ->
    if
        (hd(CH))#box.collapsed ->
            iolist_to_binary([indent(Ind+1),[flat_from_box(C) || C <-CH],"\r\n"]);
        true ->
            iolist_to_binary([[pretty_from_box(C) || C <-CH]])
    end;
pretty_from_box(#box{ind=Ind,name=Name,children=CH}) ->
    if
        (hd(CH))#box.collapsed ->
            iolist_to_binary([indent(Ind),binary_to_list(Name),"\r\n",
                indent(Ind+1),[flat_from_box(C) || C <-CH],"\r\n"]);
        true ->
            iolist_to_binary([indent(Ind), Name,"\r\n",[pretty_from_box(C) || C <-CH]])
    end;
pretty_from_box([Box|Boxes]) ->
    iolist_to_binary([pretty_from_box(Box), pretty_from_box(Boxes)]).

indent(Ind) -> lists:duplicate(Ind*2,32).

boxed_from_pt(ParseTree) ->
    try
        case foldb(ParseTree) of
            {error, _} = Error -> Error;
            Res -> {ok, Res}
        end
    catch
        _:R -> {error, {R, erlang:get_stacktrace()}}
    end.

%% Operator Binding Power -------------------------------

binding(A) when is_binary(A) -> 190;
binding('prior') -> 185;
binding('fun') -> 180;
binding('||') -> 175;
binding('*') -> 170;
binding('/') -> 170;
binding('+') -> 160;
binding('-') -> 160;
binding('=') -> 150;
binding('<=') -> 150;
binding('>=') -> 150;
binding('<>') -> 150;
binding('<') -> 150;
binding('>') -> 150;
binding('like') -> 150;
binding('is') -> 150;
binding('between') -> 150;
% binding('lists') -> 145;
binding('in') -> 140;
% binding('list') -> 135;
binding('not') -> 130;
binding('and') -> 120;
binding('or') -> 100;
binding('as') -> 90;
binding('where') -> 88;
binding('hierarchical query') -> 88;
binding('start with') -> 88;
binding('connect by') -> 88;
binding('list') -> 85;
binding('fields') -> 80;
binding('into') -> 80;
binding('hints') -> 80;
binding('opt') -> 80;
binding('from') -> 80;
% binding('where') -> 80;
binding('order by') -> 80;
binding('group by') -> 80;
binding('having') -> 80;
binding('select') -> 70;
binding('union') -> 30;
binding('union all') -> 30;
binding('all') -> 30;           %% needed here?
binding('minus') -> 30;
binding('intersect') -> 30;
binding({A,_}) -> binding(A);
binding({A,_,_}) -> binding(A);
binding({A,_,_,_}) -> binding(A);
binding(undefined) -> -1;
binding(_) -> 0.

neItems(List) ->    %% non-empty items of as list
    lists:filter(fun(X)-> X /= empty end,List).

-spec bStr(atom() | binary()) -> binary().
bStr(A) when is_atom(A) -> atom_to_binary(A, utf8);
bStr(B) when is_binary(B) -> B;
bStr(C) ->
    ?Error("Expected atom or binary, got ~p", [C]),
    throw({error, iolist_to_binary(io_lib:format("Expected atom or binary, got ~p", [C]))}).

foldb(ParseTree) ->
    foldb(0, undefined, ParseTree).

foldb(Ind, P, {select, List}) when is_list(List) ->
    Res = [foldb(Ind+1, P, Sli) || Sli <- List],
    case check_error(Res) of
        {error, Reason} -> {error, Reason};
        _ -> mk_box(Ind, neItems(Res), select)
    end;
foldb(_, _P, {hints,<<>>})               -> empty;
foldb(_, _P, {opt,  <<>>})               -> empty;
foldb(_, _P, {into,   _ })               -> empty;
foldb(_, _P, {where,  {}})               -> empty;
foldb(_, _P, {having, {}})               -> empty;
foldb(_, _P, {'hierarchical query', {}}) -> empty;

foldb(Ind, _P, T) when is_binary(T); is_atom(T) -> mk_box(Ind, [], T);

%%TODO: Improve these ugly nested cases to check for errors.
foldb(Ind, P, {'as', {'fun',Fun,List}, Alias}) ->
    case foldb(Ind, P, {'fun',Fun,List}) of<commit_msg>added support to params in the sql box.<commit_after>-module(dderl_sqlbox).

-export([ pretty_from_pt/1
        , boxed_from_pt/1
        , flat_from_box/1
        , pretty_from_box/1
        , box_to_json/1
        , foldb/3
        ]).

-include("dderl.hrl").
-include("dderl_sqlbox.hrl").

-define(DefCollInd,10).  % First indentation level which will be collapsed by default

-spec validate_children([#box{}]) -> ok | {error, binary()}.
validate_children([]) -> ok;
validate_children([#box{}]) -> ok;
validate_children(Children) ->
    case length(lists:usort([C#box.ind || C <- Children])) of
        1 -> ok;
        _ -> {error, <<"Mixed identation error">>}
    end.

-spec flat_from_box([#box{}] | #box{}) -> binary().
flat_from_box([]) -> <<>>;
flat_from_box(#box{name= <<",">> ,children=[]}) -> <<", ">>;
flat_from_box(#box{name=Name,children=[]}) ->
    iolist_to_binary([binary_to_list(Name), " "]);
flat_from_box(#box{name= <<>>,children=CH}) ->
    iolist_to_binary([[flat_from_box(C) || C <-CH]]);
flat_from_box(#box{name=Name,children=CH}) ->
    iolist_to_binary([Name," ",[flat_from_box(C) || C <-CH]]);
flat_from_box([Box|Boxes]) ->
    iolist_to_binary([flat_from_box(Box),flat_from_box(Boxes)]).

-spec pretty_from_pt(term()) -> binary() | {error, binary()}.
pretty_from_pt(ParseTree) ->
    case foldb(ParseTree) of
        {error, Reason} -> {error, Reason};
        SqlBox -> pretty_from_box(SqlBox)
    end.

-spec pretty_from_box([#box{}] | #box{}) -> binary().
pretty_from_box([]) -> [];
pretty_from_box(#box{collapsed=true}=Box) -> flat_from_box(Box);
pretty_from_box(#box{name= <<>>,children=[]}) -> <<>>;
pretty_from_box(#box{ind=Ind,name=Name,children=[]}) ->
    iolist_to_binary([indent(Ind), Name, "\r\n"]);
pretty_from_box(#box{ind=Ind,name= <<>>,children=CH}) ->
    if
        (hd(CH))#box.collapsed ->
            iolist_to_binary([indent(Ind+1),[flat_from_box(C) || C <-CH],"\r\n"]);
        true ->
            iolist_to_binary([[pretty_from_box(C) || C <-CH]])
    end;
pretty_from_box(#box{ind=Ind,name=Name,children=CH}) ->
    if
        (hd(CH))#box.collapsed ->
            iolist_to_binary([indent(Ind),binary_to_list(Name),"\r\n",
                indent(Ind+1),[flat_from_box(C) || C <-CH],"\r\n"]);
        true ->
            iolist_to_binary([indent(Ind), Name,"\r\n",[pretty_from_box(C) || C <-CH]])
    end;
pretty_from_box([Box|Boxes]) ->
    iolist_to_binary([pretty_from_box(Box), pretty_from_box(Boxes)]).

indent(Ind) -> lists:duplicate(Ind*2,32).

boxed_from_pt(ParseTree) ->
    try
        case foldb(ParseTree) of
            {error, _} = Error -> Error;
            Res -> {ok, Res}
        end
    catch
        _:R -> {error, {R, erlang:get_stacktrace()}}
    end.

%% Operator Binding Power -------------------------------

binding(A) when is_binary(A) -> 190;
binding('param') -> 190;
binding('prior') -> 185;
binding('fun') -> 180;
binding('||') -> 175;
binding('*') -> 170;
binding('/') -> 170;
binding('+') -> 160;
binding('-') -> 160;
binding('=') -> 150;
binding('<=') -> 150;
binding('>=') -> 150;
binding('<>') -> 150;
binding('<') -> 150;
binding('>') -> 150;
binding('like') -> 150;
binding('is') -> 150;
binding('between') -> 150;
% binding('lists') -> 145;
binding('in') -> 140;
% binding('list') -> 135;
binding('not') -> 130;
binding('and') -> 120;
binding('or') -> 100;
binding('as') -> 90;
binding('where') -> 88;
binding('hierarchical query') -> 88;
binding('start with') -> 88;
binding('connect by') -> 88;
binding('list') -> 85;
binding('fields') -> 80;
binding('into') -> 80;
binding('hints') -> 80;
binding('opt') -> 80;
binding('from') -> 80;
% binding('where') -> 80;
binding('order by') -> 80;
binding('group by') -> 80;
binding('having') -> 80;
binding('select') -> 70;
binding('union') -> 30;
binding('union all') -> 30;
binding('all') -> 30;           %% needed here?
binding('minus') -> 30;
binding('intersect') -> 30;
binding({A,_}) -> binding(A);
binding({A,_,_}) -> binding(A);
binding({A,_,_,_}) -> binding(A);
binding(undefined) -> -1;
binding(_) -> 0.

neItems(List) ->    %% non-empty items of as list
    lists:filter(fun(X)-> X /= empty end,List).

-spec bStr(atom() | binary()) -> binary().
bStr(A) when is_atom(A) -> atom_to_binary(A, utf8);
bStr(B) when is_binary(B) -> B;
bStr(C) ->
    ?Error("Expected atom or binary, got ~p", [C]),
    throw({error, iolist_to_binary(io_lib:format("Expected atom or binary, got ~p", [C]))}).

foldb(ParseTree) ->
    foldb(0, undefined, ParseTree).

foldb(Ind, P, {select, List}) when is_list(List) ->
    Res = [foldb(Ind+1, P, Sli) || Sli <- List],
    case check_error(Res) of
        {error, Reason} -> {error, Reason};
        _ -> mk_box(Ind, neItems(Res), select)
    end;
foldb(_, _P, {hints,<<>>})               -> empty;
foldb(_, _P, {opt,  <<>>})               -> empty;
foldb(_, _P, {into,   _ })               -> empty;
foldb(_, _P, {where,  {}})               -> empty;
foldb(_, _P, {having, {}})               -> empty;
foldb(_, _P, {'hierarchical query', {}}) -> empty;

foldb(Ind, _P, T) when is_binary(T); is_atom(T) -> mk_box(Ind, [], T);
foldb(Ind, _P, {param, T}) -> mk_box(Ind, [], T);

%%TODO: Improve these ugly nested cases to check for errors.
foldb(Ind, P, {'as', {'fun',Fun,List}, Alias}) ->
    case foldb(Ind, P, {'fun',Fun,List}) of
{'fraction_non_alphanumeric': 0.23856178065252545, 'fraction_numerical': 0.025016646057262438, 'mean_word_length': 3.4854948805460753, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 253, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '497474', 'n_tokens_mistral': 4217, 'n_tokens_neox': 3883, 'n_words': 754}