<commit_before>% @copyright 2010-2013 Zuse Institute Berlin

%   Licensed under the Apache License, Version 2.0 (the "License");
%   you may not use this file except in compliance with the License.
%   You may obtain a copy of the License at
%
%       http://www.apache.org/licenses/LICENSE-2.0
%
%   Unless required by applicable law or agreed to in writing, software
%   distributed under the License is distributed on an "AS IS" BASIS,
%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%   See the License for the specific language governing permissions and
%   limitations under the License.

%% @author <NAME> <<EMAIL>>
%% @doc    Unit tests for src/dht_node_move.erl (slide and jump operations).
%%   dht_node_move_SUITE.erl:
%%       The regular execution of the suite.
%%   dht_node_move_proto_scheduler_SUITE.erl:
%%       Executes using the proto scheduler which serializes all messages to
%%       generate a random interleaving of messages on different channels.
%% @end
%% @version $Id$

-compile(export_all).

-include("unittest.hrl").
-include("scalaris.hrl").


-type(node_type() :: predspred | pred | node | succ).

-type(node_tuple() :: {node_type(), node_type(), node_type(), node_type()}).

-type(targetid_fun() :: fun((node_tuple()) -> ?RT:key())).

-record(slideconf, {name :: atom(),
                    node :: node_type(),
                    slide_with :: node_type(),
                    targetid ::  targetid_fun()}).

-type(slide_config_record() :: #slideconf{}).


test_cases() -> [].

groups() ->
    MoveConfig = {move_config, move_config_parameters()},
    MoveConfigInc = {move_config, move_config_parameters_incremental()},
    GroupOptions = [sequence, {repeat, 1}],
    Config = [MoveConfig | GroupOptions],
    ConfigInc = [MoveConfigInc | GroupOptions],
    SendToPredTestCases =
        [
         symm4_slide_succ_rcv_load,
         symm4_slide_pred_send_load,
         tester_symm4_slide_succ_rcv_load_timeouts_succ,
         tester_symm4_slide_succ_rcv_load_timeouts_node,
         tester_symm4_slide_pred_send_load_timeouts_pred,
         tester_symm4_slide_pred_send_load_timeouts_node
        ],
    SendToSuccTestCases =
        [
         symm4_slide_succ_send_load,
         symm4_slide_pred_rcv_load,
         tester_symm4_slide_succ_send_load_timeouts_succ,
         tester_symm4_slide_succ_send_load_timeouts_node,
         tester_symm4_slide_pred_rcv_load_timeouts_pred,
         tester_symm4_slide_pred_rcv_load_timeouts_node
        ],
    SendToBothTestCases =
        [
         test_slide_adjacent,
         test_slide_conflict
        ],
    [
      {send_to_pred, Config, SendToPredTestCases},
      {send_to_pred_incremental, ConfigInc, SendToPredTestCases},
      {send_to_succ, Config, SendToSuccTestCases},
      {send_to_succ_incremental, ConfigInc, SendToSuccTestCases},
      {send_to_both, Config, SendToBothTestCases},
      {send_to_both_incremental, ConfigInc, SendToBothTestCases}
    ]
      ++
%%         unittest_helper:create_ct_groups(test_cases(), [{tester_symm4_slide_pred_send_load_timeouts_pred_incremental, [sequence, {repeat_until_any_fail, forever}]}]).
        [].

init_per_suite(Config) ->
    unittest_helper:init_per_suite(Config).

end_per_suite(Config) ->
    _ = unittest_helper:end_per_suite(Config),
    ok.

init_per_group(Group, Config) ->
    unittest_helper:init_per_group(Group, Config).

end_per_group(Group, Config) -> unittest_helper:end_per_group(Group, Config).

init_per_testcase(_TestCase, Config) ->
    % stop ring from previous test case (it may have run into a timeout)
    unittest_helper:stop_ring(),
    {priv_dir, PrivDir} = lists:keyfind(priv_dir, 1, Config),
    GroupConfig = proplists:get_value(tc_group_properties, Config, []),
    {move_config, MoveConf} = lists:keyfind(move_config, 1, GroupConfig),
    unittest_helper:make_ring_with_ids(
      fun() -> ?RT:get_replica_keys(?RT:hash_key("0")) end,
      [{config, [{log_path, PrivDir}, {dht_node, mockup_dht_node}, {monitor_perf_interval, 0},
                 {join_lb_psv, lb_psv_simple}, {lb_psv_samples, 1}]
            ++ MoveConf}]),
    % wait for all nodes to finish their join before writing data
    unittest_helper:check_ring_size_fully_joined(4),
    %% write some data (use a function because left-over tx_timeout messages can disturb the tests):
    Pid = erlang:spawn(fun() ->
                               _ = [api_tx:write(erlang:integer_to_list(X), X) || X <- lists:seq(1, 100)]
                       end),
    util:wait_for_process_to_die(Pid),
    % wait a bit for the rm-processes to settle
    timer:sleep(500),
    Config.

end_per_testcase(_TestCase, _Config) ->
    unittest_helper:stop_ring(),
    ok.

%% @doc Sets tighter timeouts for slides
-spec move_config_parameters() -> unittest_helper:kv_opts().
move_config_parameters() ->
    [{move_use_incremental_slides, false},
     {move_wait_for_reply_timeout, 1000},
     {move_send_msg_retries, 2},
     {move_send_msg_retry_delay, 0}].

%% @doc Sets tighter timeouts for slides
-spec move_config_parameters_incremental() -> unittest_helper:kv_opts().
move_config_parameters_incremental() ->
    % note: later parameters in the list override the first ones in config
    move_config_parameters() ++
        [{move_use_incremental_slides, true},
         {move_max_transport_entries, 2},
         {move_send_msg_retry_delay, 0}].

%% @doc Test slide with successor, receiving data from it (using api_tx in the bench server).
symm4_slide_succ_rcv_load(_Config) ->
    stop_time(fun() ->
                      BenchPid = erlang:spawn(fun() -> bench:increment(10, 10000) end),
                      symm4_slide_load_test(1, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      symm4_slide_load_test(2, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      symm4_slide_load_test(3, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      symm4_slide_load_test(4, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      erlang:exit(BenchPid, 'kill'),
                      util:wait_for_process_to_die(BenchPid)
              end, "symm4_slide_succ_rcv_load"),
    unittest_helper:check_ring_load(440),
    unittest_helper:check_ring_data().
<commit_msg>dht_node_move_SUITE: move more keys in incremental slides<commit_after>% @copyright 2010-2013 Zuse Institute Berlin

%   Licensed under the Apache License, Version 2.0 (the "License");
%   you may not use this file except in compliance with the License.
%   You may obtain a copy of the License at
%
%       http://www.apache.org/licenses/LICENSE-2.0
%
%   Unless required by applicable law or agreed to in writing, software
%   distributed under the License is distributed on an "AS IS" BASIS,
%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%   See the License for the specific language governing permissions and
%   limitations under the License.

%% @author <NAME> <<EMAIL>>
%% @doc    Unit tests for src/dht_node_move.erl (slide and jump operations).
%%   dht_node_move_SUITE.erl:
%%       The regular execution of the suite.
%%   dht_node_move_proto_scheduler_SUITE.erl:
%%       Executes using the proto scheduler which serializes all messages to
%%       generate a random interleaving of messages on different channels.
%% @end
%% @version $Id$

-compile(export_all).

-include("unittest.hrl").
-include("scalaris.hrl").


-type(node_type() :: predspred | pred | node | succ).

-type(node_tuple() :: {node_type(), node_type(), node_type(), node_type()}).

-type(targetid_fun() :: fun((node_tuple()) -> ?RT:key())).

-record(slideconf, {name :: atom(),
                    node :: node_type(),
                    slide_with :: node_type(),
                    targetid ::  targetid_fun()}).

-type(slide_config_record() :: #slideconf{}).


test_cases() -> [].

groups() ->
    MoveConfig = {move_config, move_config_parameters()},
    MoveConfigInc = {move_config, move_config_parameters_incremental()},
    MoveConfigInc2 = {move_config, move_config_parameters_incremental2()},
    GroupOptions = [sequence, {repeat, 1}],
    Config = [MoveConfig | GroupOptions],
    ConfigInc = [MoveConfigInc | GroupOptions],
    ConfigInc2 = [MoveConfigInc2 | GroupOptions],
    SendToPredTestCases =
        [
         symm4_slide_succ_rcv_load,
         symm4_slide_pred_send_load,
         tester_symm4_slide_succ_rcv_load_timeouts_succ,
         tester_symm4_slide_succ_rcv_load_timeouts_node,
         tester_symm4_slide_pred_send_load_timeouts_pred,
         tester_symm4_slide_pred_send_load_timeouts_node
        ],
    SendToSuccTestCases =
        [
         symm4_slide_succ_send_load,
         symm4_slide_pred_rcv_load,
         tester_symm4_slide_succ_send_load_timeouts_succ,
         tester_symm4_slide_succ_send_load_timeouts_node,
         tester_symm4_slide_pred_rcv_load_timeouts_pred,
         tester_symm4_slide_pred_rcv_load_timeouts_node
        ],
    SendToBothTestCases =
        [
         test_slide_adjacent,
         test_slide_conflict
        ],
    [
      {send_to_pred, Config, SendToPredTestCases},
      {send_to_pred_incremental, ConfigInc, SendToPredTestCases},
      {send_to_succ, Config, SendToSuccTestCases},
      {send_to_succ_incremental, ConfigInc, SendToSuccTestCases},
      {send_to_both, Config, SendToBothTestCases},
      {send_to_both_incremental, ConfigInc2, SendToBothTestCases}
    ]
      ++
%%         unittest_helper:create_ct_groups(test_cases(), [{tester_symm4_slide_pred_send_load_timeouts_pred_incremental, [sequence, {repeat_until_any_fail, forever}]}]).
        [].

init_per_suite(Config) ->
    unittest_helper:init_per_suite(Config).

end_per_suite(Config) ->
    _ = unittest_helper:end_per_suite(Config),
    ok.

init_per_group(Group, Config) ->
    unittest_helper:init_per_group(Group, Config).

end_per_group(Group, Config) -> unittest_helper:end_per_group(Group, Config).

init_per_testcase(_TestCase, Config) ->
    % stop ring from previous test case (it may have run into a timeout)
    unittest_helper:stop_ring(),
    {priv_dir, PrivDir} = lists:keyfind(priv_dir, 1, Config),
    GroupConfig = proplists:get_value(tc_group_properties, Config, []),
    {move_config, MoveConf} = lists:keyfind(move_config, 1, GroupConfig),
    unittest_helper:make_ring_with_ids(
      fun() -> ?RT:get_replica_keys(?RT:hash_key("0")) end,
      [{config, [{log_path, PrivDir}, {dht_node, mockup_dht_node}, {monitor_perf_interval, 0},
                 {join_lb_psv, lb_psv_simple}, {lb_psv_samples, 1}]
            ++ MoveConf}]),
    % wait for all nodes to finish their join before writing data
    unittest_helper:check_ring_size_fully_joined(4),
    %% write some data (use a function because left-over tx_timeout messages can disturb the tests):
    Pid = erlang:spawn(fun() ->
                               _ = [api_tx:write(erlang:integer_to_list(X), X) || X <- lists:seq(1, 100)]
                       end),
    util:wait_for_process_to_die(Pid),
    % wait a bit for the rm-processes to settle
    timer:sleep(500),
    Config.

end_per_testcase(_TestCase, _Config) ->
    unittest_helper:stop_ring(),
    ok.

%% @doc Sets tighter timeouts for slides
-spec move_config_parameters() -> unittest_helper:kv_opts().
move_config_parameters() ->
    [{move_use_incremental_slides, false},
     {move_wait_for_reply_timeout, 1000},
     {move_send_msg_retries, 2},
     {move_send_msg_retry_delay, 0}].

%% @doc Sets tighter timeouts for slides
-spec move_config_parameters_incremental() -> unittest_helper:kv_opts().
move_config_parameters_incremental() ->
    % note: later parameters in the list override the first ones in config
    move_config_parameters() ++
        [{move_use_incremental_slides, true},
         {move_max_transport_entries, 2},
         {move_send_msg_retry_delay, 0}].

%% @doc Config like above but more entries are moved at once
-spec move_config_parameters_incremental2() -> unittest_helper:kv_opts().
move_config_parameters_incremental2() ->
    move_config_parameters_incremental() ++
        [{move_max_transport_entries, 10}].

%% @doc Test slide with successor, receiving data from it (using api_tx in the bench server).
symm4_slide_succ_rcv_load(_Config) ->
    stop_time(fun() ->
                      BenchPid = erlang:spawn(fun() -> bench:increment(10, 10000) end),
                      symm4_slide_load_test(1, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      symm4_slide_load_test(2, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      symm4_slide_load_test(3, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      symm4_slide_load_test(4, succ, "slide_succ_rcv", fun id_my_to_succ_1_100/6, ?NUM_SLIDES),
                      erlang:exit(BenchPid, 'kill'),
                      util:wait_for_process_to_die(BenchPid)
              end, "symm4_slide_succ_rcv_load"),
    unittest_helper:check_ring_load(440),
    unittest_helper:check_ring_data().

{'fraction_non_alphanumeric': 0.10367384194112725, 'fraction_numerical': 0.013919525367003879, 'mean_word_length': 3.5875785066294488, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '216298', 'n_tokens_mistral': 4545, 'n_tokens_neox': 4342, 'n_words': 866}