<commit_before>%% -------------------------------------------------------------------
%%
%% Copyright (c) 2007-2010 Basho Technologies, Inc.  All Rights Reserved.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% -------------------------------------------------------------------

-module(riak_search_raptor_backend).
-author("<NAME> <<EMAIL>>").
-behavior(riak_search_backend).

-export([start/2,stop/1,index/6,multi_index/2,delete_entry/5,
         stream/6,multi_stream/4,
         info/5,info_range/7,catalog_query/3,fold/3,is_empty/1,drop/1]).
-export([toggle_raptor_debug/0, shutdown_raptor/0]).
-export([sync/0, poke/1, raptor_status/0, get_entry_keyclock/5]).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-export([test_fold/0, test_is_empty/0, test_drop/0]).
-endif.

-include_lib("raptor/include/raptor.hrl").
-include("riak_search.hrl").

% @type state() = term().
-record(state, {partition}).

-define(MAX_HANDOFF_STREAMS, 50).
-define(FOLD_TIMEOUT,         30000).
-define(INFO_TIMEOUT,          5000).
-define(INFO_RANGE_TIMEOUT,    5000).
-define(STREAM_TIMEOUT,        5000).
-define(CATALOG_QUERY_TIMEOUT, 5000).
-define(RESULT_VEC_SZ, 1000).


%% ===================================================================
%% Search Backend API
%% ===================================================================

%% @spec start(Partition :: integer(), Config :: proplist()) ->
%%          {ok, state()} | {{error, Reason :: term()}, state()}
%% @doc Start this backend.
start(Partition, _Config) ->
    {ok, #state { partition=Partition }}.

%% @spec stop(state()) -> ok | {error, Reason :: term()}
stop(_State) ->
    ok.

index(Index, Field, Term, DocId, Props, State) ->
    Partition = to_binary(State#state.partition),
    case proplists:get_value(if_newer_keyclock, Props, undefined) of
        undefined ->
            do_index(Partition, Index, Field, term, DocId, Props);
        ObjKeyClock ->
            CurrentKeyClock = get_entry_keyclock(Partition, Index, Field, Term, DocId),
            {ObjKeyClockInt,_} = string:to_integer(ObjKeyClock),
            {CurrentKeyClockInt,_} = string:to_integer(CurrentKeyClock),
            case ObjKeyClockInt > CurrentKeyClockInt of
                true ->
                    Props1 = proplists:delete(if_newer_keyclock, Props),
                    do_index(Partition, Index, Field, Term, DocId, Props1);
                _ -> noreply
            end
    end.

do_index(Partition, Index, Field, Term, DocId, Props) ->
    case get(index_con) of
        undefined ->
            {ok, Conn} = riak_sock_pool:checkout(?CONN_POOL),
            put(index_con, Conn);
        C ->
            Conn = C
    end,
    %%try
        raptor_conn:index(Conn,
                          to_binary(Index),
                          to_binary(Field),
                          to_binary(Term),
                          to_binary(DocId),
                          Partition,
                          term_to_binary(Props),
                          current_key_clock()),
    %%after
    %%    riak_sock_pool:checkin(?CONN_POOL, Conn)
    %%end,
    noreply.

multi_index(IFTVPList, State) ->
    Partition = to_binary(State#state.partition),
    {ok, Conn} = riak_sock_pool:checkout(?CONN_POOL),
    try
        %% Index for raptor is fire and forget - no point checking
        %% return value<commit_msg>remove debug<commit_after>%% -------------------------------------------------------------------
%%
%% Copyright (c) 2007-2010 Basho Technologies, Inc.  All Rights Reserved.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% -------------------------------------------------------------------

-module(riak_search_raptor_backend).
-author("<NAME> <<EMAIL>>").
-behavior(riak_search_backend).

-export([start/2,stop/1,index/6,multi_index/2,delete_entry/5,
         stream/6,multi_stream/4,
         info/5,info_range/7,catalog_query/3,fold/3,is_empty/1,drop/1]).
-export([toggle_raptor_debug/0, shutdown_raptor/0]).
-export([sync/0, poke/1, raptor_status/0, get_entry_keyclock/5]).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-export([test_fold/0, test_is_empty/0, test_drop/0]).
-endif.

-include_lib("raptor/include/raptor.hrl").
-include("riak_search.hrl").

% @type state() = term().
-record(state, {partition}).

-define(MAX_HANDOFF_STREAMS, 50).
-define(FOLD_TIMEOUT,         30000).
-define(INFO_TIMEOUT,          5000).
-define(INFO_RANGE_TIMEOUT,    5000).
-define(STREAM_TIMEOUT,        5000).
-define(CATALOG_QUERY_TIMEOUT, 5000).
-define(RESULT_VEC_SZ, 1000).


%% ===================================================================
%% Search Backend API
%% ===================================================================

%% @spec start(Partition :: integer(), Config :: proplist()) ->
%%          {ok, state()} | {{error, Reason :: term()}, state()}
%% @doc Start this backend.
start(Partition, _Config) ->
    {ok, #state { partition=Partition }}.

%% @spec stop(state()) -> ok | {error, Reason :: term()}
stop(_State) ->
    ok.

index(Index, Field, Term, DocId, Props, State) ->
    Partition = to_binary(State#state.partition),
    case proplists:get_value(if_newer_keyclock, Props, undefined) of
        undefined ->
            do_index(Partition, Index, Field, term, DocId, Props);
        ObjKeyClock ->
            CurrentKeyClock = get_entry_keyclock(Partition, Index, Field, Term, DocId),
            {ObjKeyClockInt,_} = string:to_integer(ObjKeyClock),
            {CurrentKeyClockInt,_} = string:to_integer(CurrentKeyClock),
            case ObjKeyClockInt > CurrentKeyClockInt of
                true ->
                    Props1 = proplists:delete(if_newer_keyclock, Props),
                    do_index(Partition, Index, Field, Term, DocId, Props1);
                _ -> noreply
            end
    end.

do_index(Partition, Index, Field, Term, DocId, Props) ->
    {ok, Conn} = riak_sock_pool:checkout(?CONN_POOL),
    try
        raptor_conn:index(Conn,
                          to_binary(Index),
                          to_binary(Field),
                          to_binary(Term),
                          to_binary(DocId),
                          Partition,
                          term_to_binary(Props),
                          current_key_clock())
    after
        riak_sock_pool:checkin(?CONN_POOL, Conn)
    end,
    noreply.

multi_index(IFTVPList, State) ->
    Partition = to_binary(State#state.partition),
    {ok, Conn} = riak_sock_pool:checkout(?CONN_POOL),
    try
        %% Index for raptor is fire and forget - no point checking
        %% return value
{'fraction_non_alphanumeric': 0.19379642903688257, 'fraction_numerical': 0.01642121725531083, 'mean_word_length': 2.5250344510794673, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5155528', 'n_tokens_mistral': 2380, 'n_tokens_neox': 2261, 'n_words': 520}