<commit_before>-module(dql).

-export([prepare/1]).

-ifdef(TEST).
-export([parse/1]).
-endif.

-export_type([query_part/0, dqe_fun/0, query_stmt/0]).

-type time() :: #{ op => time, args => [pos_integer() | ms | s | m | h | d | w]} | pos_integer().

-type relative_time() :: time() |
                         now |
                         #{ op => 'after' | before, args => [pos_integer() | time()]} |
                         #{op => ago, ags => [time()]}.


-type range() :: #{ op => between, args => [relative_time() | relative_time()]} |
                 #{ op => last, args => [time()]}.

-type sig_element() :: string | number | metric.

-type dqe_fun() ::
        #{
           op => fcall,
           signature => [sig_element()],
           args => #{}
         }.

-type get_stmt() ::
        #{op => get, args => [[binary()] | non_neg_integer()]} |
        #{op => sget, args => [[binary() | '*'] | non_neg_integer()]}.


-type query_part() :: cmb_stmt() | flat_stmt().
-type cmb_stmt() ::
        {combine, dqe_fun(), [statement()] | get_stmt()}.

-type statement() ::
        get_stmt() |
        dqe_fun() |
        cmb_stmt().

-type flat_terminal() ::
        {combine, dqe_fun(), [flat_stmt() | get_stmt()]}.

-type flat_stmt() ::
        {calc, [dqe_fun()], flat_terminal() | get_stmt()}.

-type named() :: #{op => named, args => [binary() | flat_stmt()]}.
-type query_stmt() :: {named, binary(), flat_stmt()}.

-type parser_error() ::
        {error, binary()}.

-type alias() :: term().

%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc Takes a query, parses it and prepares it for the query
%% engine to execute.
%% @end
%%--------------------------------------------------------------------
-spec prepare(string()) ->
                     {error, term()} |
                     {ok, [query_stmt()]}.
prepare(S) ->
    case parse(S) of
        {ok, {select, Qs, Aliases, T}} ->
            dqe_lib:pdebug('parse', "Query parsed: ~s", [S]),
            extract_aliases(Qs, T, Aliases);
        E ->
            E
    end.

%%%===================================================================
%%% Parsing steps
%%%===================================================================

%%--------------------------------------------------------------------
%% @private
%% @doc Extrat the alises from all query parts
%% @end
%%--------------------------------------------------------------------
-spec extract_aliases([statement()], time(), [term()]) ->
                     {error, term()} |
                     {ok, [query_stmt()]}.
extract_aliases(Qs, T, Aliases) ->
    AliasesF =
        lists:foldl(fun({alias, Alias, Res}, AAcc) ->
                            gb_trees:enter(Alias, Res, AAcc)
                    end, gb_trees:empty(), Aliases),
    dqe_lib:pdebug('parse', "Aliases resolved.", []),
    resolve_aliases(Qs, T, AliasesF).

%%--------------------------------------------------------------------
%% @private
%% @doc Resolves the aliases.
%% @end
%%--------------------------------------------------------------------
-spec resolve_aliases([statement()], time(), gb_trees:tree()) ->
                        {error, term()} |
                        {ok, [query_stmt()]}.
resolve_aliases(Qs, T, Aliases) ->
    {QQ, _AliasesQ} =
        lists:foldl(fun(Q, {QAcc, AAcc}) ->
                            {Q1, A1} = resolve_aliases(Q, AAcc),
                            {[Q1 | QAcc] , A1}
                    end, {[], Aliases}, Qs),
    dqe_lib:pdebug('parse', "Preprocessor done.", []),
    QQ1 = lists:reverse(QQ),
    resolve_query_functions(QQ1, T).

%%--------------------------------------------------------------------
%% @private
%% @doc Apply types and resolve the functions called in the query
%% to dqe_fun functions.
%% @end
%%--------------------------------------------------------------------
-spec resolve_query_functions([statement()], time()) ->
                     {error, term()} |
                     {ok, [query_stmt()]}.
resolve_query_functions(Qs, T) ->
    case resolve_functions(Qs, []) of
        {ok, Qs1} ->
            flatten_step(Qs1, T);
        E ->
            E
    end.

%%--------------------------------------------------------------------
%% @private
%% @doc Flattens the tree into a list of flat queries.
%% @end
%%--------------------------------------------------------------------
-spec flatten_step([statement()], time()) ->
                          {ok, [query_stmt()]}.
flatten_step(Qs, T) ->
    Qs1 = [flatten(Q) || Q <- Qs],
    dqe_lib:pdebug('parse', "Query flattened.", []),
    expand(Qs1, T).

%%--------------------------------------------------------------------
%% @private
%% @doc Expand lookup and glob.
%% @end
%%--------------------------------------------------------------------
-spec expand([flat_stmt()], time()) ->
                    {'ok',[query_stmt()]}.
expand(Qs, T) ->
    Qs1 = [expand(Q) || Q <- Qs],
    Qs2 = lists:flatten(Qs1),
    get_resolution(Qs2, T).

%%--------------------------------------------------------------------
%% @private
%% @doc Fetch the resolution for each sub query.
%% @end
%%--------------------------------------------------------------------
-spec get_resolution([flat_stmt()], time()) ->
                            %% {error, term()} |
                            {'ok',[{named, binary(), flat_stmt()}]}.
get_resolution(Qs, T) ->
    {Qs1, _, _} = lists:foldl(fun get_resolution_fn/2,
                           {[], T, #{}}, Qs),
    Qs2 = lists:reverse(Qs1),
    propagate_resolutions(Qs2).

%%--------------------------------------------------------------------
%% @private
%% @doc calculate resolutions for the whole call stack.
%% @end
%%--------------------------------------------------------------------
propagate_resolutions(Qs) ->
    Qs1 = [apply_times(Q) || Q <- Qs],
    {ok, Qs1}.

%%%===================================================================
%%% Internal functions
%%%===================================================================

%%--------------------------------------------------------------------
%% @private
%% @doc Run a string through the lexer and parser and return somewhat
%% sensible errors if requried.
%% @end
%%--------------------------------------------------------------------
-spec parse(string() | binary()) ->
                   parser_error() |
                   {ok, {select, [statement()], [alias()], range()}}.

parse(S) when is_binary(S)->
    parse(binary_to_list(S));

parse(S) ->
    case dql_lexer:string(S) of
        {error,{Line, dql_lexer,E},1} ->
            lexer_error(Line, E);
        {ok, L, _} ->
            case dql_parser:parse(L) of
                {error, {Line, dql_parser, E}} ->
                    parser_error(Line, E);
                R ->
                    R
            end
    end.

parser_error(Line, E)  ->
    {error, list_to_binary(io_lib:format("Parser error in line ~p: ~s",
                                         [Line, E]))}.

lexer_error(Line, {illegal, E})  ->
    {error, list_to_binary(io_lib:format("Lexer error in line ~p illegal: ~s",
                                         [Line, E]))};

lexer_error(Line, E)  ->
    {error, list_to_binary(io_lib:format("Lexer error in line ~p: ~s",
                                         [Line, E]))}.


%%--------------------------------------------------------------------
%% @private
%% @doc Fetch the resulution for a single sub query.
%% @end
%%--------------------------------------------------------------------
-spec get_resolution_fn(named(),
                        {[named()], time(), #{}}) ->
                               {[named()], time(), #{}}.
get_resolution_fn(Q, {QAcc, T, #{} = RAcc}) when is_list(QAcc) ->
    {ok, Q1, RAcc1} = get_times(Q, T, RAcc),
    {[Q1 | QAcc], T, RAcc1}.

%%--------------------------------------------------------------------
%% @private
%% @doc Resolves the aliases
%% @end
%%--------------------------------------------------------------------
resolve_aliases(O = #{op   := fcall,
                     args := Args = #{inputs := Input}},
               Aliases) ->
    {Input1, A1} =
        lists:foldl(fun (Q, {QAcc, AAcc}) ->
                            {Qx, Ax} = resolve_aliases(Q, AAcc),
                            {[Qx | QAcc], Ax}
                    end, {[], Aliases}, Input),
    Input2 = lists:reverse(Input1),
    {O#{args => Args#{inputs => Input2}}, A1};
resolve_aliases(O = #{op := named, args := [N, Q]}, Aliases) ->
    {Q1, A1} = resolve_aliases(Q, Aliases),
    {O#{args => [N, Q1]}, A1};
resolve_aliases(#{op := var, args := [V]}, Aliases) ->
    {value, G} = gb_trees:lookup(V, Aliases),
    {G, Aliases};
resolve_aliases(O = #{}, Aliases) ->
    {O, Aliases};
resolve_aliases(N, A) when is_number(N)->
    {N, A}.

%%--------------------------------------------------------------------
%% @private
%% @doc Resolves functions based on their parameter types
%% @end
%%--------------------------------------------------------------------
resolve_functions(#{op := named, args := [N, Q], return := undefined}) ->
    case resolve_functions(Q) of
        {ok, R = #{return := T}} ->
            {ok, #{
               op => named,
               args => [N, R],
               signature => [string, T],
               return => T
              }
            };
        E ->
            E
    end;

resolve_functions(#{op := fcall, args := #{name   := Function,
                                           inputs := Args}}) ->
    case resolve_functions(Args, []) of
        {ok, Args1} ->
            %% Determine the type of each argument
            Types = [T || #{return := T} <- Args1],
            %% Decide wather an argument is a constant (passed to init)
            %% or a nested function
            Args2 = [{C, is_constant(T)} || C = #{return := T} <- Args1],
            Constants = [C || {C, true} <- Args2],
            Inputs = [C || {C, false} <- Args2],
            %% Lookup if we know a function with the given types
            case dqe_fun:lookup(Function, Types) of
                %% If we find a function that does not take a
                %% list of sub functions we know this is a normal
                %%aggregate.
                {ok,{{_, none, _}, ReturnType, FunMod}} ->
                    FArgs = #{name      => Function,
                              orig_args => Args,
                              mod       => FunMod,
                              inputs    => Inputs,
                              constants => Constants},
                    {ok, #{
                       op => fcall,
                       args => FArgs,
                       signature => Types,
                       return => ReturnType
                      }};
                %% If we find one that takes a list of sub functions
                %% we know it is a combinator function.
                {ok,{{_, _, _}, ReturnType, FunMod}} ->
                    FArgs = #{name      => Function,
                              orig_args => Args,
                              mod       => FunMod,
                              inputs    => Inputs,
                              constants => Constants},
                    {ok, #{<commit_msg>Fix funciton detection<commit_after>-module(dql).

-export([prepare/1]).

-ifdef(TEST).
-export([parse/1]).
-endif.

-export_type([query_part/0, dqe_fun/0, query_stmt/0]).

-type time() :: #{ op => time, args => [pos_integer() | ms | s | m | h | d | w]} | pos_integer().

-type relative_time() :: time() |
                         now |
                         #{ op => 'after' | before, args => [pos_integer() | time()]} |
                         #{op => ago, ags => [time()]}.


-type range() :: #{ op => between, args => [relative_time() | relative_time()]} |
                 #{ op => last, args => [time()]}.

-type sig_element() :: string | number | metric.

-type dqe_fun() ::
        #{
           op => fcall,
           signature => [sig_element()],
           args => #{}
         }.

-type get_stmt() ::
        #{op => get, args => [[binary()] | non_neg_integer()]} |
        #{op => sget, args => [[binary() | '*'] | non_neg_integer()]}.


-type query_part() :: cmb_stmt() | flat_stmt().
-type cmb_stmt() ::
        {combine, dqe_fun(), [statement()] | get_stmt()}.

-type statement() ::
        get_stmt() |
        dqe_fun() |
        cmb_stmt().

-type flat_terminal() ::
        {combine, dqe_fun(), [flat_stmt() | get_stmt()]}.

-type flat_stmt() ::
        {calc, [dqe_fun()], flat_terminal() | get_stmt()}.

-type named() :: #{op => named, args => [binary() | flat_stmt()]}.
-type query_stmt() :: {named, binary(), flat_stmt()}.

-type parser_error() ::
        {error, binary()}.

-type alias() :: term().

%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc Takes a query, parses it and prepares it for the query
%% engine to execute.
%% @end
%%--------------------------------------------------------------------
-spec prepare(string()) ->
                     {error, term()} |
                     {ok, [query_stmt()]}.
prepare(S) ->
    case parse(S) of
        {ok, {select, Qs, Aliases, T}} ->
            dqe_lib:pdebug('parse', "Query parsed: ~s", [S]),
            extract_aliases(Qs, T, Aliases);
        E ->
            E
    end.

%%%===================================================================
%%% Parsing steps
%%%===================================================================

%%--------------------------------------------------------------------
%% @private
%% @doc Extrat the alises from all query parts
%% @end
%%--------------------------------------------------------------------
-spec extract_aliases([statement()], time(), [term()]) ->
                     {error, term()} |
                     {ok, [query_stmt()]}.
extract_aliases(Qs, T, Aliases) ->
    AliasesF =
        lists:foldl(fun({alias, Alias, Res}, AAcc) ->
                            gb_trees:enter(Alias, Res, AAcc)
                    end, gb_trees:empty(), Aliases),
    dqe_lib:pdebug('parse', "Aliases resolved.", []),
    resolve_aliases(Qs, T, AliasesF).

%%--------------------------------------------------------------------
%% @private
%% @doc Resolves the aliases.
%% @end
%%--------------------------------------------------------------------
-spec resolve_aliases([statement()], time(), gb_trees:tree()) ->
                        {error, term()} |
                        {ok, [query_stmt()]}.
resolve_aliases(Qs, T, Aliases) ->
    {QQ, _AliasesQ} =
        lists:foldl(fun(Q, {QAcc, AAcc}) ->
                            {Q1, A1} = resolve_aliases(Q, AAcc),
                            {[Q1 | QAcc] , A1}
                    end, {[], Aliases}, Qs),
    dqe_lib:pdebug('parse', "Preprocessor done.", []),
    QQ1 = lists:reverse(QQ),
    resolve_query_functions(QQ1, T).

%%--------------------------------------------------------------------
%% @private
%% @doc Apply types and resolve the functions called in the query
%% to dqe_fun functions.
%% @end
%%--------------------------------------------------------------------
-spec resolve_query_functions([statement()], time()) ->
                     {error, term()} |
                     {ok, [query_stmt()]}.
resolve_query_functions(Qs, T) ->
    case resolve_functions(Qs, []) of
        {ok, Qs1} ->
            flatten_step(Qs1, T);
        E ->
            E
    end.

%%--------------------------------------------------------------------
%% @private
%% @doc Flattens the tree into a list of flat queries.
%% @end
%%--------------------------------------------------------------------
-spec flatten_step([statement()], time()) ->
                          {ok, [query_stmt()]}.
flatten_step(Qs, T) ->
    Qs1 = [flatten(Q) || Q <- Qs],
    dqe_lib:pdebug('parse', "Query flattened.", []),
    expand(Qs1, T).

%%--------------------------------------------------------------------
%% @private
%% @doc Expand lookup and glob.
%% @end
%%--------------------------------------------------------------------
-spec expand([flat_stmt()], time()) ->
                    {'ok',[query_stmt()]}.
expand(Qs, T) ->
    Qs1 = [expand(Q) || Q <- Qs],
    Qs2 = lists:flatten(Qs1),
    get_resolution(Qs2, T).

%%--------------------------------------------------------------------
%% @private
%% @doc Fetch the resolution for each sub query.
%% @end
%%--------------------------------------------------------------------
-spec get_resolution([flat_stmt()], time()) ->
                            %% {error, term()} |
                            {'ok',[{named, binary(), flat_stmt()}]}.
get_resolution(Qs, T) ->
    {Qs1, _, _} = lists:foldl(fun get_resolution_fn/2,
                           {[], T, #{}}, Qs),
    Qs2 = lists:reverse(Qs1),
    propagate_resolutions(Qs2).

%%--------------------------------------------------------------------
%% @private
%% @doc calculate resolutions for the whole call stack.
%% @end
%%--------------------------------------------------------------------
propagate_resolutions(Qs) ->
    Qs1 = [apply_times(Q) || Q <- Qs],
    {ok, Qs1}.

%%%===================================================================
%%% Internal functions
%%%===================================================================

%%--------------------------------------------------------------------
%% @private
%% @doc Run a string through the lexer and parser and return somewhat
%% sensible errors if requried.
%% @end
%%--------------------------------------------------------------------
-spec parse(string() | binary()) ->
                   parser_error() |
                   {ok, {select, [statement()], [alias()], range()}}.

parse(S) when is_binary(S)->
    parse(binary_to_list(S));

parse(S) ->
    case dql_lexer:string(S) of
        {error,{Line, dql_lexer,E},1} ->
            lexer_error(Line, E);
        {ok, L, _} ->
            case dql_parser:parse(L) of
                {error, {Line, dql_parser, E}} ->
                    parser_error(Line, E);
                R ->
                    R
            end
    end.

parser_error(Line, E)  ->
    {error, list_to_binary(io_lib:format("Parser error in line ~p: ~s",
                                         [Line, E]))}.

lexer_error(Line, {illegal, E})  ->
    {error, list_to_binary(io_lib:format("Lexer error in line ~p illegal: ~s",
                                         [Line, E]))};

lexer_error(Line, E)  ->
    {error, list_to_binary(io_lib:format("Lexer error in line ~p: ~s",
                                         [Line, E]))}.


%%--------------------------------------------------------------------
%% @private
%% @doc Fetch the resulution for a single sub query.
%% @end
%%--------------------------------------------------------------------
-spec get_resolution_fn(named(),
                        {[named()], time(), #{}}) ->
                               {[named()], time(), #{}}.
get_resolution_fn(Q, {QAcc, T, #{} = RAcc}) when is_list(QAcc) ->
    {ok, Q1, RAcc1} = get_times(Q, T, RAcc),
    {[Q1 | QAcc], T, RAcc1}.

%%--------------------------------------------------------------------
%% @private
%% @doc Resolves the aliases
%% @end
%%--------------------------------------------------------------------
resolve_aliases(O = #{op   := fcall,
                     args := Args = #{inputs := Input}},
               Aliases) ->
    {Input1, A1} =
        lists:foldl(fun (Q, {QAcc, AAcc}) ->
                            {Qx, Ax} = resolve_aliases(Q, AAcc),
                            {[Qx | QAcc], Ax}
                    end, {[], Aliases}, Input),
    Input2 = lists:reverse(Input1),
    {O#{args => Args#{inputs => Input2}}, A1};
resolve_aliases(O = #{op := named, args := [N, Q]}, Aliases) ->
    {Q1, A1} = resolve_aliases(Q, Aliases),
    {O#{args => [N, Q1]}, A1};
resolve_aliases(#{op := var, args := [V]}, Aliases) ->
    {value, G} = gb_trees:lookup(V, Aliases),
    {G, Aliases};
resolve_aliases(O = #{}, Aliases) ->
    {O, Aliases};
resolve_aliases(N, A) when is_number(N)->
    {N, A}.

%%--------------------------------------------------------------------
%% @private
%% @doc Resolves functions based on their parameter types
%% @end
%%--------------------------------------------------------------------
resolve_functions(#{op := named, args := [N, Q], return := undefined}) ->
    case resolve_functions(Q) of
        {ok, R = #{return := T}} ->
            {ok, #{
               op => named,
               args => [N, R],
               signature => [string, T],
               return => T
              }
            };
        E ->
            E
    end;

resolve_functions(#{op := fcall, args := #{name   := Function,
                                           inputs := Args}}) ->
    case resolve_functions(Args, []) of
        {ok, Args1} ->
            %% Determine the type of each argument
            Types = [T || #{return := T} <- Args1],
            %% Decide wather an argument is a constant (passed to init)
            %% or a nested function
            Args2 = [{C, is_constant(T)} || C = #{return := T} <- Args1],
            Constants = [C || {C, true} <- Args2],
            Inputs = [C || {C, false} <- Args2],
            %% Lookup if we know a function with the given types
            case dqe_fun:lookup(Function, Types) of
                %% If we find a function that does not take a
                %% list of sub functions we know this is a normal
                %%aggregate.
                {ok,{{_, _, none}, ReturnType, FunMod}} ->
                    FArgs = #{name      => Function,
                              orig_args => Args,
                              mod       => FunMod,
                              inputs    => Inputs,
                              constants => Constants},
                    {ok, #{
                       op => fcall,
                       args => FArgs,
                       signature => Types,
                       return => ReturnType
                      }};
                %% If we find one that takes a list of sub functions
                %% we know it is a combinator function.
                {ok,{{_, _, _}, ReturnType, FunMod}} ->
                    FArgs = #{name      => Function,
                              orig_args => Args,
                              mod       => FunMod,
                              inputs    => Inputs,
                              constants => Constants},
                    {ok, #{
{'fraction_non_alphanumeric': 0.3242579683458284, 'fraction_numerical': 0.004121180235871805, 'mean_word_length': 1.2281918530819576, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5027674', 'n_tokens_mistral': 6218, 'n_tokens_neox': 6037, 'n_words': 1496}