<commit_before>%%%-------------------------------------------------------------------
%%%
%%% riak_kv_qry_worker: Riak SQL per-query workers
%%%
%%% Copyright (C) 2015 Basho Technologies, Inc. All rights reserved
%%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%%
%%%-------------------------------------------------------------------

%% @doc Under the queue manager accepting raw parsed and lexed queries
%%      from the user, workers take individual queries and communicate
%%      with eleveldb backend to execute the queries (with
%%      sub-queries), and hold the results until fetched back to the
%%      user.

-module(riak_kv_qry_worker).

-behaviour(gen_server).

%% OTP API
-export([start_link/1]).

%% gen_server callbacks
-export([
         init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3
        ]).

-ifdef(TEST).
-export([
         runner_TEST/1
        ]).
-endif.

-include_lib("riak_ql/include/riak_ql_ddl.hrl").

-define(NO_SIDEEFFECTS, []).
-define(NO_MAX_RESULTS, no_max_results).
-define(NO_PG_SORT, undefined).

-record(state, {
          name                 :: atom(),
          ddl                  :: undefined | #ddl_v1{},
          qry      = none      :: none | #riak_sql_v1{},
          qid      = undefined :: undefined | {node(), non_neg_integer()},
          sub_qrys  = []       :: [integer()],
          status    = void     :: void | accumulating_chunks,
          receiver_pid         :: pid(),
          result    = []       :: [{non_neg_integer(), list()}] | [{binary(), term()}]
         }).

%%%===================================================================
%%% OTP API
%%%===================================================================
-spec start_link(RegisteredName::atom()) -> {ok, pid()} | ignore | {error, term()}.
start_link(RegisteredName) ->
    gen_server:start_link({local, RegisteredName}, ?MODULE, [RegisteredName], []).

%%%===================================================================
%%% gen_server callbacks
%%%===================================================================

-spec init([RegisteredName::atom()]) -> {ok, #state{}}.
%% @private
init([RegisteredName]) ->
    pop_next_query(),
    {ok, new_state(RegisteredName)}.

-spec handle_call(term(), {pid(), term()}, #state{}) ->
                         {reply, Reply::ok | {error, atom()} | list(), #state{}}.
%% @private
handle_call(Request, _, State) ->
    case handle_req(Request, State) of
        {{error, _} = Error, _SEs, NewState} ->
            {reply, Error, NewState};
        {Reply, SEs, NewState} ->
            ok = handle_side_effects(SEs),
            {reply, Reply, NewState}
    end.


-spec handle_cast(term(), #state{}) -> {noreply, #state{}}.
%% @private
handle_cast(Msg, State) ->
    lager:info("Not handling cast message ~p", [Msg]),
    {noreply, State}.

%% @private
-spec handle_info(term(), #state{}) -> {noreply, #state{}}.
handle_info(pop_next_query, State1) ->
    {query, ReceivePid, QId, Qry, DDL} = riak_kv_qry_queue:blocking_pop(),
    Request = {execute, {QId, Qry, DDL}},
    case handle_req(Request, State1) of
        {{error, _} = Error, _, State2} ->
            ReceivePid ! Error,
            {noreply, new_state(State2#state.name)};
        {ok, SEs, State2} ->
            ok = handle_side_effects(SEs),
            {noreply, State2#state{ receiver_pid = ReceivePid }}
    end;

handle_info({{SubQId, QId}, done},
            State = #state{qid       = QId,
                           receiver_pid = ReceiverPid,
                           result    = IndexedChunks,
                           sub_qrys  = SubQQ}) ->
    lager:debug("Received done on QId ~p (~p); SubQQ: ~p", [QId, SubQId, SubQQ]),
    case SubQQ of
        [] ->
            lager:debug("Done collecting on QId ~p (~p): ~p", [QId, SubQId, IndexedChunks]),
            %% sort by index, to reassemble according to coverage plan
            {_, R2} = lists:unzip(lists:sort(IndexedChunks)),
            Results = lists:append(R2),
            % send the results to the waiting client process
            ReceiverPid ! {ok, Results},
            pop_next_query(),
            %% drop indexes, serialize
            {noreply, new_state(State#state.name)};
        _MoreSubQueriesNotDone ->
            {noreply, State}
    end;

handle_info({{SubQId, QId}, {results, Chunk}},
            State = #state{qid      = QId,
                           qry      = Qry,
                           result   = IndexedChunks,
                           sub_qrys = SubQs}) ->
    #riak_sql_v1{'SELECT' = SelectSpec} = Qry,
    NewS = case lists:member(SubQId, SubQs) of
               true ->
                   Decoded = decode_results(lists:flatten(Chunk), SelectSpec),
                   lager:debug("Got chunk on QId ~p (~p); SubQQ: ~p", [QId, SubQId, SubQs]),
                   NSubQ = lists:delete(SubQId, SubQs),
                   State#state{status   = accumulating_chunks,
                               result   = [{SubQId, Decoded} | IndexedChunks],
                               sub_qrys = NSubQ};
               false ->
                   %% discard;
                   %% Don't touch state as it may have already 'finished'.
                   State
           end,
    {noreply, NewS};

handle_info({{SubQId, QId}, {error, timeout}},
            State = #state{receiver_pid = ReceiverPid,
                           qid    = QId,
                           result = IndexedChunks}) ->
    lager:warning("Backend timed out while collecting on QId ~p (~p);"
                  " dropping ~b chunks of data accumulated so far",
                  [QId, SubQId, length(IndexedChunks)]),
    ReceiverPid ! {error, backend_timeout},
    pop_next_query(),
    {noreply, new_state(State#state.name)};

handle_info({{_SubQId, QId1}, _}, State = #state{qid = QId2}) when QId1 =/= QId2 ->
    %% catches late results or errors such getting results for invalid QIds.
    lager:warning("Bad query id ~p (expected ~p)", [QId1, QId2]),
    {noreply, State}.
<commit_msg>Reduce lager warning to debug per Jon<commit_after>%%%-------------------------------------------------------------------
%%%
%%% riak_kv_qry_worker: Riak SQL per-query workers
%%%
%%% Copyright (C) 2015 Basho Technologies, Inc. All rights reserved
%%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%%
%%%-------------------------------------------------------------------

%% @doc Under the queue manager accepting raw parsed and lexed queries
%%      from the user, workers take individual queries and communicate
%%      with eleveldb backend to execute the queries (with
%%      sub-queries), and hold the results until fetched back to the
%%      user.

-module(riak_kv_qry_worker).

-behaviour(gen_server).

%% OTP API
-export([start_link/1]).

%% gen_server callbacks
-export([
         init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3
        ]).

-ifdef(TEST).
-export([
         runner_TEST/1
        ]).
-endif.

-include_lib("riak_ql/include/riak_ql_ddl.hrl").

-define(NO_SIDEEFFECTS, []).
-define(NO_MAX_RESULTS, no_max_results).
-define(NO_PG_SORT, undefined).

-record(state, {
          name                 :: atom(),
          ddl                  :: undefined | #ddl_v1{},
          qry      = none      :: none | #riak_sql_v1{},
          qid      = undefined :: undefined | {node(), non_neg_integer()},
          sub_qrys  = []       :: [integer()],
          status    = void     :: void | accumulating_chunks,
          receiver_pid         :: pid(),
          result    = []       :: [{non_neg_integer(), list()}] | [{binary(), term()}]
         }).

%%%===================================================================
%%% OTP API
%%%===================================================================
-spec start_link(RegisteredName::atom()) -> {ok, pid()} | ignore | {error, term()}.
start_link(RegisteredName) ->
    gen_server:start_link({local, RegisteredName}, ?MODULE, [RegisteredName], []).

%%%===================================================================
%%% gen_server callbacks
%%%===================================================================

-spec init([RegisteredName::atom()]) -> {ok, #state{}}.
%% @private
init([RegisteredName]) ->
    pop_next_query(),
    {ok, new_state(RegisteredName)}.

-spec handle_call(term(), {pid(), term()}, #state{}) ->
                         {reply, Reply::ok | {error, atom()} | list(), #state{}}.
%% @private
handle_call(Request, _, State) ->
    case handle_req(Request, State) of
        {{error, _} = Error, _SEs, NewState} ->
            {reply, Error, NewState};
        {Reply, SEs, NewState} ->
            ok = handle_side_effects(SEs),
            {reply, Reply, NewState}
    end.


-spec handle_cast(term(), #state{}) -> {noreply, #state{}}.
%% @private
handle_cast(Msg, State) ->
    lager:info("Not handling cast message ~p", [Msg]),
    {noreply, State}.

%% @private
-spec handle_info(term(), #state{}) -> {noreply, #state{}}.
handle_info(pop_next_query, State1) ->
    {query, ReceivePid, QId, Qry, DDL} = riak_kv_qry_queue:blocking_pop(),
    Request = {execute, {QId, Qry, DDL}},
    case handle_req(Request, State1) of
        {{error, _} = Error, _, State2} ->
            ReceivePid ! Error,
            {noreply, new_state(State2#state.name)};
        {ok, SEs, State2} ->
            ok = handle_side_effects(SEs),
            {noreply, State2#state{ receiver_pid = ReceivePid }}
    end;

handle_info({{SubQId, QId}, done},
            State = #state{qid       = QId,
                           receiver_pid = ReceiverPid,
                           result    = IndexedChunks,
                           sub_qrys  = SubQQ}) ->
    lager:debug("Received done on QId ~p (~p); SubQQ: ~p", [QId, SubQId, SubQQ]),
    case SubQQ of
        [] ->
            lager:debug("Done collecting on QId ~p (~p): ~p", [QId, SubQId, IndexedChunks]),
            %% sort by index, to reassemble according to coverage plan
            {_, R2} = lists:unzip(lists:sort(IndexedChunks)),
            Results = lists:append(R2),
            % send the results to the waiting client process
            ReceiverPid ! {ok, Results},
            pop_next_query(),
            %% drop indexes, serialize
            {noreply, new_state(State#state.name)};
        _MoreSubQueriesNotDone ->
            {noreply, State}
    end;

handle_info({{SubQId, QId}, {results, Chunk}},
            State = #state{qid      = QId,
                           qry      = Qry,
                           result   = IndexedChunks,
                           sub_qrys = SubQs}) ->
    #riak_sql_v1{'SELECT' = SelectSpec} = Qry,
    NewS = case lists:member(SubQId, SubQs) of
               true ->
                   Decoded = decode_results(lists:flatten(Chunk), SelectSpec),
                   lager:debug("Got chunk on QId ~p (~p); SubQQ: ~p", [QId, SubQId, SubQs]),
                   NSubQ = lists:delete(SubQId, SubQs),
                   State#state{status   = accumulating_chunks,
                               result   = [{SubQId, Decoded} | IndexedChunks],
                               sub_qrys = NSubQ};
               false ->
                   %% discard;
                   %% Don't touch state as it may have already 'finished'.
                   State
           end,
    {noreply, NewS};

handle_info({{SubQId, QId}, {error, timeout}},
            State = #state{receiver_pid = ReceiverPid,
                           qid    = QId,
                           result = IndexedChunks}) ->
    lager:warning("Backend timed out while collecting on QId ~p (~p);"
                  " dropping ~b chunks of data accumulated so far",
                  [QId, SubQId, length(IndexedChunks)]),
    ReceiverPid ! {error, backend_timeout},
    pop_next_query(),
    {noreply, new_state(State#state.name)};

handle_info({{_SubQId, QId1}, _}, State = #state{qid = QId2}) when QId1 =/= QId2 ->
    %% catches late results or errors such getting results for invalid QIds.
    lager:debug("Bad query id ~p (expected ~p)", [QId1, QId2]),
    {noreply, State}.

{'fraction_non_alphanumeric': 0.19153104279686559, 'fraction_numerical': 0.004972875226039783, 'mean_word_length': 2.1715651135005976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2943933', 'n_tokens_mistral': 4023, 'n_tokens_neox': 3850, 'n_words': 1010}