<commit_before>-module(dql).
-export([prepare/1, parse/1, execute/1, unparse/1, glob_match/2]).
-ignore_xref([prepare/1, parse/1, execute/1, unparse/1, glob_match/2]).

parse(S) when is_binary(S)->
    parse(binary_to_list(S));
parse(S) ->
    case dql_lexer:string(S) of
        {error,{Line,dql_lexer,E},1} ->
            {error, io_lib:format("Error in line ~p: ~p", [Line, E])};
        {ok, L, _} ->
            case dql_parser:parse(L) of
                {error, {Line, dql_parser, E}} ->
                    {error, io_lib:format("Error in line ~p: ~s", [Line, E])};
                R ->
                    R
            end
    end.

prepare(S) ->
    case parse(S) of
        {ok, {select, Qx, Aliasesx, Tx, Rx}} ->
            prepare(Qx, Aliasesx, Tx, Rx);
        {ok, {select, Qx, Tx, Rx}} ->
            prepare(Qx, [], Tx, Rx);
        E ->
            E
    end.

prepare(Qs, Aliases, T, R) ->
    Rms = to_ms(R),
    T1 = apply_times(T, Rms),
    {_AF, AliasesF, MetricsF} =
        lists:foldl(fun({alias, Alias, Resolution}, {QAcc, AAcc, MAcc}) ->
                            {Q1, A1, M1} = preprocess_qry(Resolution, AAcc, MAcc, Rms),
                            {[Q1 | QAcc], gb_trees:enter(Alias, Resolution, A1), M1}
                    end, {[], gb_trees:empty(), gb_trees:empty()}, Aliases),
    {QQ, AliasesQ, MetricsQ} =
        lists:foldl(fun(Q, {QAcc, AAcc, MAcc}) ->
                            {Q1, A1, M1} = preprocess_qry(Q, AAcc, MAcc, Rms),
                            {[Q1 | QAcc] , A1, M1}
                    end, {[], AliasesF, MetricsF}, Qs),
    {Start, Count} = compute_se(T1, Rms),
    {QQ, Start, Count, Rms, AliasesQ, MetricsQ}.

compute_se({between, S, E}, _Rms) when E > S->
    {S, E - S};
compute_se({between, S, E}, _Rms) ->
    {E, S - E};

compute_se({last, N}, Rms) ->
    _Now = {Mega, Sec, Micro} = now(),
    NowMs = ((Mega * 1000000  + Sec) * 1000000 + Micro) div 1000,
    %%UTC = calendar:now_to_universal_time(Now),
    %%UTCs = calendar:datetime_to_gregorian_seconds(UTC) - 62167219200,
    %%UTCms = (UTCs * 1000) + (NowMs rem 1000),
    RelativeNow = NowMs div Rms,
    {RelativeNow - N, N}.


preprocess_qry({named, N, Q}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{named, N, Q1}, A1, M1};

preprocess_qry({aggr, AggF, Q}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{aggr, AggF, Q1}, A1, M1};

preprocess_qry({aggr, AggF, Q, T}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{aggr, AggF, Q1, T}, A1, M1};

preprocess_qry({aggr, AggF, Q, Arg, T}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{aggr, AggF, Q1, Arg, T}, A1, M1};

preprocess_qry({math, MathF, Q, V}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{math, MathF, Q1, V}, A1, M1};


preprocess_qry({get, BM}, Aliases, Metrics, _Rms) ->
    Metrics1 = case gb_trees:lookup(BM, Metrics) of
                   none ->
                       gb_trees:insert(BM, {get, 0}, Metrics);
                   {value, {get, N}} ->
                       gb_trees:update(BM, {get, N + 1}, Metrics)
               end,
    {{get, BM}, Aliases, Metrics1};

preprocess_qry({mget, AggrF, BM}, Aliases, Metrics, _Rms) ->
    Metrics1 = case gb_trees:lookup({AggrF, BM}, Metrics) of
                   none ->
                       gb_trees:insert({AggrF, BM}, {mget, 0}, Metrics);
                   {value, {get, N}} ->
                       gb_trees:update({AggrF, BM}, {mget, N + 1}, Metrics)
               end,
    {{mget, AggrF, BM}, Aliases, Metrics1};

preprocess_qry({var, V}, Aliases, Metrics, _Rms) ->
    Metrics1 = case gb_trees:lookup(V, Aliases) of
                   none ->
                       Metrics;
                   {value, {get, BM}} ->
                       case gb_trees:lookup(BM, Metrics) of
                           none ->
                               gb_trees:insert(BM, {get, 1}, Metrics);
                           {value, {get, N}} ->
                               gb_trees:update(BM, {get, N + 1}, Metrics)
                       end
               end,
    {{var, V}, Aliases, Metrics1};

preprocess_qry(Q, A, M, _) ->
    {Q, A, M}.

execute(Qry) ->
    case prepare(Qry) of
        {Qs, S, C, Rms, A, M} ->
            {D, _} = lists:foldl(fun({named, Name, Q}, {RAcc, MAcc}) ->
                                         {{D, R}, M1} = execute(Q, S, C, Rms, A, MAcc),
                                         {[{Name, R, D} | RAcc], M1};
                                     (Q, {RAcc, MAcc}) ->
                                         {{D, R}, M1} = execute(Q, S, C, Rms, A, MAcc),
                                         {[{unparse(Q), R, D} | RAcc], M1}
                                 end, {[], M}, Qs),
            {ok, D};
        E ->
            E
    end.

execute({named, _, Q}, S, C, Rms, A, M) ->
    execute(Q, S, C, Rms, A, M);

execute({aggr, derivate, Q}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    {{mmath_aggr:derivate(D), Res}, M1};

execute({aggr, avg, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:avg(D, T1), T1}, M1};

execute({aggr, sum, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:sum(D, T1), T1}, M1};

execute({aggr, max, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:max(D, T1), T1}, M1};

execute({aggr, empty, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:empty(D, T1), T1}, M1};

execute({aggr, min, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:min(D, T1), T1}, M1};

execute({aggr, percentile, Q, P, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:percentile(D, P, T1), T1}, M1};

execute({math, multiply, Q, V}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    {{mmath_aggr:scale(D, V), Res}, M1};

execute({math, divide, Q, V}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    {{mmath_aggr:scale(D, 1/V), Res}, M1};



execute({get, BM = {B, M}}, S, C, _Rms, _A, Metrics) ->
    case gb_trees:get(BM, Metrics) of
        {get, N} when N =< 1 ->
            {ok, D} = dalmatiner_connection:get(B, M, S, C),
            {{D, 1}, gb_trees:delete(BM, Metrics)};
        {get, N} -><commit_msg>Fixed wrong order in percentile calls.<commit_after>-module(dql).
-export([prepare/1, parse/1, execute/1, unparse/1, glob_match/2]).
-ignore_xref([prepare/1, parse/1, execute/1, unparse/1, glob_match/2]).

parse(S) when is_binary(S)->
    parse(binary_to_list(S));
parse(S) ->
    case dql_lexer:string(S) of
        {error,{Line,dql_lexer,E},1} ->
            {error, io_lib:format("Error in line ~p: ~p", [Line, E])};
        {ok, L, _} ->
            case dql_parser:parse(L) of
                {error, {Line, dql_parser, E}} ->
                    {error, io_lib:format("Error in line ~p: ~s", [Line, E])};
                R ->
                    R
            end
    end.

prepare(S) ->
    case parse(S) of
        {ok, {select, Qx, Aliasesx, Tx, Rx}} ->
            prepare(Qx, Aliasesx, Tx, Rx);
        {ok, {select, Qx, Tx, Rx}} ->
            prepare(Qx, [], Tx, Rx);
        E ->
            E
    end.

prepare(Qs, Aliases, T, R) ->
    Rms = to_ms(R),
    T1 = apply_times(T, Rms),
    {_AF, AliasesF, MetricsF} =
        lists:foldl(fun({alias, Alias, Resolution}, {QAcc, AAcc, MAcc}) ->
                            {Q1, A1, M1} = preprocess_qry(Resolution, AAcc, MAcc, Rms),
                            {[Q1 | QAcc], gb_trees:enter(Alias, Resolution, A1), M1}
                    end, {[], gb_trees:empty(), gb_trees:empty()}, Aliases),
    {QQ, AliasesQ, MetricsQ} =
        lists:foldl(fun(Q, {QAcc, AAcc, MAcc}) ->
                            {Q1, A1, M1} = preprocess_qry(Q, AAcc, MAcc, Rms),
                            {[Q1 | QAcc] , A1, M1}
                    end, {[], AliasesF, MetricsF}, Qs),
    {Start, Count} = compute_se(T1, Rms),
    {QQ, Start, Count, Rms, AliasesQ, MetricsQ}.

compute_se({between, S, E}, _Rms) when E > S->
    {S, E - S};
compute_se({between, S, E}, _Rms) ->
    {E, S - E};

compute_se({last, N}, Rms) ->
    _Now = {Mega, Sec, Micro} = now(),
    NowMs = ((Mega * 1000000  + Sec) * 1000000 + Micro) div 1000,
    %%UTC = calendar:now_to_universal_time(Now),
    %%UTCs = calendar:datetime_to_gregorian_seconds(UTC) - 62167219200,
    %%UTCms = (UTCs * 1000) + (NowMs rem 1000),
    RelativeNow = NowMs div Rms,
    {RelativeNow - N, N}.


preprocess_qry({named, N, Q}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{named, N, Q1}, A1, M1};

preprocess_qry({aggr, AggF, Q}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{aggr, AggF, Q1}, A1, M1};

preprocess_qry({aggr, AggF, Q, T}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{aggr, AggF, Q1, T}, A1, M1};

preprocess_qry({aggr, AggF, Q, Arg, T}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{aggr, AggF, Q1, Arg, T}, A1, M1};

preprocess_qry({math, MathF, Q, V}, Aliases, Metrics, Rms) ->
    {Q1, A1, M1} = preprocess_qry(Q, Aliases, Metrics, Rms),
    {{math, MathF, Q1, V}, A1, M1};


preprocess_qry({get, BM}, Aliases, Metrics, _Rms) ->
    Metrics1 = case gb_trees:lookup(BM, Metrics) of
                   none ->
                       gb_trees:insert(BM, {get, 0}, Metrics);
                   {value, {get, N}} ->
                       gb_trees:update(BM, {get, N + 1}, Metrics)
               end,
    {{get, BM}, Aliases, Metrics1};

preprocess_qry({mget, AggrF, BM}, Aliases, Metrics, _Rms) ->
    Metrics1 = case gb_trees:lookup({AggrF, BM}, Metrics) of
                   none ->
                       gb_trees:insert({AggrF, BM}, {mget, 0}, Metrics);
                   {value, {get, N}} ->
                       gb_trees:update({AggrF, BM}, {mget, N + 1}, Metrics)
               end,
    {{mget, AggrF, BM}, Aliases, Metrics1};

preprocess_qry({var, V}, Aliases, Metrics, _Rms) ->
    Metrics1 = case gb_trees:lookup(V, Aliases) of
                   none ->
                       Metrics;
                   {value, {get, BM}} ->
                       case gb_trees:lookup(BM, Metrics) of
                           none ->
                               gb_trees:insert(BM, {get, 1}, Metrics);
                           {value, {get, N}} ->
                               gb_trees:update(BM, {get, N + 1}, Metrics)
                       end
               end,
    {{var, V}, Aliases, Metrics1};

preprocess_qry(Q, A, M, _) ->
    {Q, A, M}.

execute(Qry) ->
    case prepare(Qry) of
        {Qs, S, C, Rms, A, M} ->
            {D, _} = lists:foldl(fun({named, Name, Q}, {RAcc, MAcc}) ->
                                         {{D, R}, M1} = execute(Q, S, C, Rms, A, MAcc),
                                         {[{Name, R, D} | RAcc], M1};
                                     (Q, {RAcc, MAcc}) ->
                                         {{D, R}, M1} = execute(Q, S, C, Rms, A, MAcc),
                                         {[{unparse(Q), R, D} | RAcc], M1}
                                 end, {[], M}, Qs),
            {ok, D};
        E ->
            E
    end.

execute({named, _, Q}, S, C, Rms, A, M) ->
    execute(Q, S, C, Rms, A, M);

execute({aggr, derivate, Q}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    {{mmath_aggr:derivate(D), Res}, M1};

execute({aggr, avg, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:avg(D, T1), T1}, M1};

execute({aggr, sum, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:sum(D, T1), T1}, M1};

execute({aggr, max, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:max(D, T1), T1}, M1};

execute({aggr, empty, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:empty(D, T1), T1}, M1};

execute({aggr, min, Q, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:min(D, T1), T1}, M1};

execute({aggr, percentile, Q, P, T}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    T1 = apply_times(T, Rms * Res),
    {{mmath_aggr:percentile(D, T1, P), T1}, M1};

execute({math, multiply, Q, V}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    {{mmath_aggr:scale(D, V), Res}, M1};

execute({math, divide, Q, V}, S, C, Rms, A, M) ->
    {{D, Res}, M1} = execute(Q, S, C, Rms, A, M),
    {{mmath_aggr:scale(D, 1/V), Res}, M1};



execute({get, BM = {B, M}}, S, C, _Rms, _A, Metrics) ->
    case gb_trees:get(BM, Metrics) of
        {get, N} when N =< 1 ->
            {ok, D} = dalmatiner_connection:get(B, M, S, C),
            {{D, 1}, gb_trees:delete(BM, Metrics)};
        {get, N} ->
{'fraction_non_alphanumeric': 0.19436495707682147, 'fraction_numerical': 0.02157164869029276, 'mean_word_length': 1.9305525693399268, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4106365', 'n_tokens_mistral': 5705, 'n_tokens_neox': 5473, 'n_words': 1435}