<commit_before>% @copyright 2012 Zuse Institute Berlin

%   Licensed under the Apache License, Version 2.0 (the "License");
%   you may not use this file except in compliance with the License.
%   You may obtain a copy of the License at
%
%       http://www.apache.org/licenses/LICENSE-2.0
%
%   Unless required by applicable law or agreed to in writing, software
%   distributed under the License is distributed on an "AS IS" BASIS,
%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%   See the License for the specific language governing permissions and
%   limitations under the License.

%% @author <NAME> <<EMAIL>>
%% @doc Administrative helper functions for replica repair evaluation
%% @version $Id:  $
-module(rr_admin).

-export([make_ring/2, 
         fill_ring/3,
         start_sync/0,
         db_stats/0,
         set_recon_method/1]).

-export([test_upd/0,
         test_regen/0]). %TEST SETS

-include("scalaris.hrl").

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% TYPES
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-type ring_type()    :: symmetric | random.
-type failure_type() :: update | regen | mixed.
-type db_type()      :: wiki | random.
-type db_parameter() :: {ftype, failure_type()} |
                        {fprob, 0..100} |            %failure probability
                        {distribution, db_generator:distribution()}.
-record(db_status, {entries    = 0  :: non_neg_integer(),
                    existing   = 0  :: non_neg_integer(),
                    missing    = 0  :: non_neg_integer(),
                    outdated   = 0  :: non_neg_integer()}).
-type db_status() :: #db_status{}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-define(DBSizeKey, rr_admin_dbsize).    %Process Dictionary Key for generated db size
-define(ReplicationFactor, 4).

-define(IIF(C, A, B), case C of
                          true -> A;
                          _ -> B
                      end).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Eval Quick Start
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec test_upd() -> ok.
test_upd() ->
    make_ring(symmetric, 10),
    R = fill_ring(random, 10000, [{ftype, mixed}]),
    print_db_status(R),
    db_stats().

-spec test_regen() -> ok.
test_regen() ->
    make_ring(symmetric, 10),
    R = fill_ring(random, 10000, [{ftype, regen}]),
    print_db_status(R),
    start_sync(),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% API Functions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec make_ring(ring_type(), pos_integer()) -> ok.
make_ring(Type, Size) ->
    %if ring exists kill all   
    _ = set_recon_method(bloom), 
    _ = case Type of
            random -> 
                _ = admin:add_node([{first}]),
                admin:add_nodes(Size -1);
            symmetric ->
                Ids = get_symmetric_ids(Size),
                _ = admin:add_node([{first}, {{dht_node, id}, hd(Ids)}]),
                [admin:add_node_at_id(Id) || Id <- tl(Ids)]
        end,
    wait_for_stable_ring(),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% @doc  DBSize=Number of Data Entities in DB (without replicas)
-spec fill_ring(db_type(), pos_integer(), [db_parameter()]) -> db_status().
fill_ring(Type, DBSize, Params) ->
    erlang:put(?DBSizeKey, ?ReplicationFactor * DBSize),
    case Type of
        random -> fill_random(DBSize, Params);
        wiki -> fill_wiki(DBSize, Params)
    end.

fill_random(DBSize, Params) ->    
    Distr = proplists:get_value(distribution, Params, uniform),            
    I = hd(intervals:split(intervals:all(), ?ReplicationFactor)),
    Keys = db_generator:get_db(I, DBSize, Distr),    
    insert_random_db(Keys, Params).
fill_wiki(DBSize, Params) ->
    %TODO
    #db_status{}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec start_sync() -> ok.
start_sync() ->
    Nodes = get_node_list(),
    io:format("--> START SYNC - ~p~n", [config:read(rep_update_recon_method)]),
    %start
    lists:foreach(fun(Node) ->
                          comm:send(Node, {send_to_group_member, rep_upd, {rep_update_trigger}})
                  end, 
                  Nodes),
    %wait for end
    lists:foreach(
      fun(Node) -> 
              util:wait_for(
                fun() -> 
                        comm:send(Node, {send_to_group_member, rep_upd, {get_state, comm:this(), open_sync}}),
                        receive
                            {get_state_response, Val} -> Val =:= 0
                        end
                end)
      end, 
      Nodes),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec db_stats() -> ok.
db_stats() ->
    S = get_db_status(),
    print_db_status(S),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec set_recon_method(rr_recon:method()) -> ok | {error, term()}.
set_recon_method(Method) ->
    config:write(rrepair_enabled, true),
    config:write(rep_update_interval, 100000000),
    config:write(rep_update_trigger, trigger_periodic),
    config:write(rep_update_recon_method, Method),
    config:write(rep_update_resolve_method, simple),<commit_msg>- rr_admin: rrepair process name fixed<commit_after>% @copyright 2012 Zuse Institute Berlin

%   Licensed under the Apache License, Version 2.0 (the "License");
%   you may not use this file except in compliance with the License.
%   You may obtain a copy of the License at
%
%       http://www.apache.org/licenses/LICENSE-2.0
%
%   Unless required by applicable law or agreed to in writing, software
%   distributed under the License is distributed on an "AS IS" BASIS,
%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%   See the License for the specific language governing permissions and
%   limitations under the License.

%% @author <NAME> <<EMAIL>>
%% @doc Administrative helper functions for replica repair evaluation
%% @version $Id:  $
-module(rr_admin).

-export([make_ring/2, 
         fill_ring/3,
         start_sync/0,
         db_stats/0,
         set_recon_method/1]).

-export([test_upd/0,
         test_regen/0]). %TEST SETS

-include("scalaris.hrl").

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% TYPES
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-type ring_type()    :: symmetric | random.
-type failure_type() :: update | regen | mixed.
-type db_type()      :: wiki | random.
-type db_parameter() :: {ftype, failure_type()} |
                        {fprob, 0..100} |            %failure probability
                        {distribution, db_generator:distribution()}.
-record(db_status, {entries    = 0  :: non_neg_integer(),
                    existing   = 0  :: non_neg_integer(),
                    missing    = 0  :: non_neg_integer(),
                    outdated   = 0  :: non_neg_integer()}).
-type db_status() :: #db_status{}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-define(DBSizeKey, rr_admin_dbsize).    %Process Dictionary Key for generated db size
-define(ReplicationFactor, 4).

-define(IIF(C, A, B), case C of
                          true -> A;
                          _ -> B
                      end).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Eval Quick Start
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec test_upd() -> ok.
test_upd() ->
    make_ring(symmetric, 10),
    _ = fill_ring(random, 10000, [{ftype, update}]),
    db_stats(),
    util:for_to(1, 10, 
                fun(_) ->
                        start_sync(),
                        db_stats()
                end),
    ok.    

-spec test_regen() -> ok.
test_regen() ->
    make_ring(symmetric, 10),
    R = fill_ring(random, 10000, [{ftype, regen}]),
    print_db_status(R),
    start_sync(),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% API Functions
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec make_ring(ring_type(), pos_integer()) -> ok.
make_ring(Type, Size) ->
    %if ring exists kill all   
    _ = set_recon_method(bloom), 
    _ = case Type of
            random -> 
                _ = admin:add_node([{first}]),
                admin:add_nodes(Size -1);
            symmetric ->
                Ids = get_symmetric_ids(Size),
                _ = admin:add_node([{first}, {{dht_node, id}, hd(Ids)}]),
                [admin:add_node_at_id(Id) || Id <- tl(Ids)]
        end,
    wait_for_stable_ring(),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% @doc  DBSize=Number of Data Entities in DB (without replicas)
-spec fill_ring(db_type(), pos_integer(), [db_parameter()]) -> db_status().
fill_ring(Type, DBSize, Params) ->
    erlang:put(?DBSizeKey, ?ReplicationFactor * DBSize),
    case Type of
        random -> fill_random(DBSize, Params);
        wiki -> fill_wiki(DBSize, Params)
    end.

fill_random(DBSize, Params) ->    
    Distr = proplists:get_value(distribution, Params, uniform),            
    I = hd(intervals:split(intervals:all(), ?ReplicationFactor)),
    Keys = db_generator:get_db(I, DBSize, Distr),    
    insert_random_db(Keys, Params).
fill_wiki(DBSize, Params) ->
    %TODO
    #db_status{}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec start_sync() -> ok.
start_sync() ->
    Nodes = get_node_list(),
    io:format("--> START SYNC - ~p~n", [config:read(rep_update_recon_method)]),
    %start
    lists:foreach(fun(Node) ->
                          comm:send(Node, {send_to_group_member, rrepair, {rr_trigger}})
                  end, 
                  Nodes),
    %wait for end
    lists:foreach(
      fun(Node) -> 
              util:wait_for(
                fun() -> 
                        comm:send(Node, {send_to_group_member, rrepair, {get_state, comm:this(), open_sync}}),
                        receive
                            {get_state_response, Val} -> Val =:= 0
                        end
                end)
      end, 
      Nodes),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec db_stats() -> ok.
db_stats() ->
    S = get_db_status(),
    print_db_status(S),
    ok.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec set_recon_method(rr_recon:method()) -> ok | {error, term()}.
set_recon_method(Method) ->
    config:write(rrepair_enabled, true),
    config:write(rep_update_interval, 100000000),
    config:write(rep_update_trigger, trigger_periodic),
    config:write(rep_update_recon_method, Method),
    config:write(rep_update_resolve_method, simple),
{'fraction_non_alphanumeric': 0.26763094061629683, 'fraction_numerical': 0.009073757973227922, 'mean_word_length': 2.851903114186851, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2817178', 'n_tokens_mistral': 3342, 'n_tokens_neox': 3091, 'n_words': 705}