<commit_before>%% ---------------------------------------------------------------------
%%
%% Copyright (c) 2007-2013 Basho Technologies, Inc.  All Rights Reserved.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% ---------------------------------------------------------------------
-module(rtcs).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("erlcloud/include/erlcloud_aws.hrl").

-import(rt, [join/2,
             wait_until_nodes_ready/1,
             wait_until_no_pending_changes/1]).

-define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])).
-define(DEV(N), list_to_atom(?DEVS(N))).
-define(CSDEVS(N), lists:concat(["rcs-dev", N, "@127.0.0.1"])).
-define(CSDEV(N), list_to_atom(?CSDEVS(N))).

-define(RIAK_ROOT, build_paths.root).
-define(RIAK_CURRENT, build_paths.current).
-define(EE_ROOT, build_paths.ee_root).
-define(EE_CURRENT, build_paths.ee_current).
-define(CS_ROOT, build_paths.cs_root).
-define(CS_CURRENT, build_paths.cs_current).
-define(STANCHION_ROOT, build_paths.stanchion_root).
-define(STANCHION_CURRENT, build_paths.stanchion_current).

-define(PROXY_HOST, "localhost").
-define(S3_HOST, "s3.amazonaws.com").
-define(S3_PORT, 80).
-define(DEFAULT_PROTO, "http").

setup(NumNodes) ->
    setup(NumNodes, default_configs()).

setup(NumNodes, Configs) ->
    %% Start the erlcloud app
    erlcloud:start(),

    %% STFU sasl
    application:load(sasl),
    application:set_env(sasl, sasl_error_logger, false),

    Cfgs = configs(Configs),
    {RiakNodes, _CSNodes, _Stanchion} = Nodes = deploy_nodes(NumNodes, Cfgs),
    rt:wait_until_nodes_ready(RiakNodes),
    lager:info("Make cluster"),
    rtcs:make_cluster(RiakNodes),
    rt:wait_until_ring_converged(RiakNodes),
    {AdminKeyId, AdminSecretKey} = setup_admin_user(NumNodes, Cfgs),
    AdminConfig = config(AdminKeyId,
                         AdminSecretKey,
                         rtcs:cs_port(hd(RiakNodes))),
    {AdminConfig, Nodes}.

setup2x2() ->
    setup2x2(default_configs()).

setup2x2(Configs) ->
    %% Start the erlcloud app
    erlcloud:start(),

    %% STFU sasl
    application:load(sasl),
    application:set_env(sasl, sasl_error_logger, false),

    Cfgs = configs(Configs),
    lager:info("Configs = ~p", [ Cfgs]),
    {RiakNodes, _CSNodes, _Stanchion} = Nodes = deploy_nodes(4, Cfgs),
    rt:wait_until_nodes_ready(RiakNodes),
    lager:info("Make cluster"),
    rtcs:make_2x2_clusters(RiakNodes),
    rt:wait_until_ring_converged(RiakNodes),
    {AdminKeyId, AdminSecretKey} = setup_admin_user(4, Cfgs),
    AdminConfig = rtcs:config(AdminKeyId,
                              AdminSecretKey,
                              rtcs:cs_port(hd(RiakNodes))),
    {AdminConfig, Nodes}.


configs(CustomConfigs) ->
    [{riak, proplists:get_value(riak, CustomConfigs, riak_config())},
     {cs, proplists:get_value(cs, CustomConfigs, cs_config())},
     {stanchion, proplists:get_value(stanchion,
                                     CustomConfigs,
                                     stanchion_config())}].

default_configs() ->
    [{riak, riak_config()},
     {stanchion, stanchion_config()},
     {cs, cs_config()}].

config(Key, Secret, Port) ->
    erlcloud_s3:new(Key,
                    Secret,
                    ?S3_HOST,
                    Port, % inets issue precludes using ?S3_PORT
                    ?DEFAULT_PROTO,
                    ?PROXY_HOST,
                    Port,
                    []).

create_user(Node, UserIndex) ->
    {A, B, C} = erlang:now(),
    User = "Test User" ++ integer_to_list(UserIndex),
    Email = lists:flatten(io_lib:format("~p~p~p@bash<EMAIL>", [A, B, C])),
    {KeyId, Secret, _Id} = create_user(cs_port(Node), Email, User),
    lager:info("Created user ~p with keys ~p ~p", [Email, KeyId, Secret]),
    {KeyId, Secret}.

create_admin_user(Node) ->
    User = "admin",
    Email = "<EMAIL>",
    {KeyId, Secret, Id} = create_user(cs_port(Node), Email, User),
    lager:info("Riak CS Admin account created with ~p",[Email]),
    lager:info("KeyId = ~p",[KeyId]),
    lager:info("KeySecret = ~p",[Secret]),
    lager:info("Id = ~p",[Id]),
    {KeyId, Secret}.

cs_port(Node) ->
    8070 + rt_cs_dev:node_id(Node).

riak_config() ->
    riak_config(
      rt_config:get(build_type, oss),
      rt_config:get(backend, {multi_backend, bitcask})).

riak_config(oss, Backend) ->
    riak_oss_config(Backend);
riak_config(ee, Backend) ->
    riak_ee_config(Backend).

riak_oss_config(Backend) ->
    CSCurrent = rt_config:get(build_paths.cs_current),
    AddPaths = filelib:wildcard(CSCurrent ++ "/dev/dev1/lib/riak_cs*/ebin"),
    [
     lager_config(),
     {riak_core,
      [{default_bucket_props, [{allow_mult, true}]}]},
     {riak_kv,
      [{add_paths, AddPaths}] ++
          backend_config(Backend)
      }
    ].

backend_config(memory) ->
      [{storage_backend, riak_kv_memory_backend}];
backend_config({multi_backend, BlocksBackend}) ->
      [
       {storage_backend, riak_cs_kv_multi_backend},
       {multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]},
       {multi_backend_default, be_default},
       {multi_backend,
        [{be_default, riak_kv_eleveldb_backend,
          [
           {max_open_files, 20},
           {data_root, "./leveldb"}
          ]},
         blocks_backend_config(BlocksBackend)
        ]}
      ].<commit_msg>Set pb_backlog in Riak config for riak_test runs<commit_after>%% ---------------------------------------------------------------------
%%
%% Copyright (c) 2007-2013 Basho Technologies, Inc.  All Rights Reserved.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
%% except in compliance with the License.  You may obtain
%% a copy of the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing,
%% software distributed under the License is distributed on an
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
%% KIND, either express or implied.  See the License for the
%% specific language governing permissions and limitations
%% under the License.
%%
%% ---------------------------------------------------------------------
-module(rtcs).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("erlcloud/include/erlcloud_aws.hrl").

-import(rt, [join/2,
             wait_until_nodes_ready/1,
             wait_until_no_pending_changes/1]).

-define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])).
-define(DEV(N), list_to_atom(?DEVS(N))).
-define(CSDEVS(N), lists:concat(["rcs-dev", N, "@127.0.0.1"])).
-define(CSDEV(N), list_to_atom(?CSDEVS(N))).

-define(RIAK_ROOT, build_paths.root).
-define(RIAK_CURRENT, build_paths.current).
-define(EE_ROOT, build_paths.ee_root).
-define(EE_CURRENT, build_paths.ee_current).
-define(CS_ROOT, build_paths.cs_root).
-define(CS_CURRENT, build_paths.cs_current).
-define(STANCHION_ROOT, build_paths.stanchion_root).
-define(STANCHION_CURRENT, build_paths.stanchion_current).

-define(PROXY_HOST, "localhost").
-define(S3_HOST, "s3.amazonaws.com").
-define(S3_PORT, 80).
-define(DEFAULT_PROTO, "http").

setup(NumNodes) ->
    setup(NumNodes, default_configs()).

setup(NumNodes, Configs) ->
    %% Start the erlcloud app
    erlcloud:start(),

    %% STFU sasl
    application:load(sasl),
    application:set_env(sasl, sasl_error_logger, false),

    Cfgs = configs(Configs),
    {RiakNodes, _CSNodes, _Stanchion} = Nodes = deploy_nodes(NumNodes, Cfgs),
    rt:wait_until_nodes_ready(RiakNodes),
    lager:info("Make cluster"),
    rtcs:make_cluster(RiakNodes),
    rt:wait_until_ring_converged(RiakNodes),
    {AdminKeyId, AdminSecretKey} = setup_admin_user(NumNodes, Cfgs),
    AdminConfig = config(AdminKeyId,
                         AdminSecretKey,
                         rtcs:cs_port(hd(RiakNodes))),
    {AdminConfig, Nodes}.

setup2x2() ->
    setup2x2(default_configs()).

setup2x2(Configs) ->
    %% Start the erlcloud app
    erlcloud:start(),

    %% STFU sasl
    application:load(sasl),
    application:set_env(sasl, sasl_error_logger, false),

    Cfgs = configs(Configs),
    lager:info("Configs = ~p", [ Cfgs]),
    {RiakNodes, _CSNodes, _Stanchion} = Nodes = deploy_nodes(4, Cfgs),
    rt:wait_until_nodes_ready(RiakNodes),
    lager:info("Make cluster"),
    rtcs:make_2x2_clusters(RiakNodes),
    rt:wait_until_ring_converged(RiakNodes),
    {AdminKeyId, AdminSecretKey} = setup_admin_user(4, Cfgs),
    AdminConfig = rtcs:config(AdminKeyId,
                              AdminSecretKey,
                              rtcs:cs_port(hd(RiakNodes))),
    {AdminConfig, Nodes}.


configs(CustomConfigs) ->
    [{riak, proplists:get_value(riak, CustomConfigs, riak_config())},
     {cs, proplists:get_value(cs, CustomConfigs, cs_config())},
     {stanchion, proplists:get_value(stanchion,
                                     CustomConfigs,
                                     stanchion_config())}].

default_configs() ->
    [{riak, riak_config()},
     {stanchion, stanchion_config()},
     {cs, cs_config()}].

config(Key, Secret, Port) ->
    erlcloud_s3:new(Key,
                    Secret,
                    ?S3_HOST,
                    Port, % inets issue precludes using ?S3_PORT
                    ?DEFAULT_PROTO,
                    ?PROXY_HOST,
                    Port,
                    []).

create_user(Node, UserIndex) ->
    {A, B, C} = erlang:now(),
    User = "Test User" ++ integer_to_list(UserIndex),
    Email = lists:flatten(io_lib:format("~p~p~<EMAIL>", [A, B, C])),
    {KeyId, Secret, _Id} = create_user(cs_port(Node), Email, User),
    lager:info("Created user ~p with keys ~p ~p", [Email, KeyId, Secret]),
    {KeyId, Secret}.

create_admin_user(Node) ->
    User = "admin",
    Email = "<EMAIL>",
    {KeyId, Secret, Id} = create_user(cs_port(Node), Email, User),
    lager:info("Riak CS Admin account created with ~p",[Email]),
    lager:info("KeyId = ~p",[KeyId]),
    lager:info("KeySecret = ~p",[Secret]),
    lager:info("Id = ~p",[Id]),
    {KeyId, Secret}.

cs_port(Node) ->
    8070 + rt_cs_dev:node_id(Node).

riak_config() ->
    riak_config(
      rt_config:get(build_type, oss),
      rt_config:get(backend, {multi_backend, bitcask})).

riak_config(oss, Backend) ->
    riak_oss_config(Backend);
riak_config(ee, Backend) ->
    riak_ee_config(Backend).

riak_oss_config(Backend) ->
    CSCurrent = rt_config:get(build_paths.cs_current),
    AddPaths = filelib:wildcard(CSCurrent ++ "/dev/dev1/lib/riak_cs*/ebin"),
    [
     lager_config(),
     {riak_core,
      [{default_bucket_props, [{allow_mult, true}]}]},
     {riak_api,
      [{pb_backlog, 256}]},
     {riak_kv,
      [{add_paths, AddPaths}] ++
          backend_config(Backend)
      }
    ].

backend_config(memory) ->
      [{storage_backend, riak_kv_memory_backend}];
backend_config({multi_backend, BlocksBackend}) ->
      [
       {storage_backend, riak_cs_kv_multi_backend},
       {multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]},
       {multi_backend_default, be_default},
       {multi_backend,
        [{be_default, riak_kv_eleveldb_backend,
          [
           {max_open_files, 20},
           {data_root, "./leveldb"}
          ]},
         blocks_backend_config(BlocksBackend)
        ]}
      ].
{'fraction_non_alphanumeric': 0.16656802232180604, 'fraction_numerical': 0.009216200219835968, 'mean_word_length': 2.947930574098798, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4974234', 'n_tokens_mistral': 4174, 'n_tokens_neox': 4057, 'n_words': 732}