<commit_before>%%% -*- erlang-indent-level: 4 -*-
%%%-------------------------------------------------------------------
%%% @copyright (C) 2017, <NAME>
%%% @doc Keeping state of running nodes during system test.
%%% @end
%%% --------------------------------------------------------------------

-module(aest_nodes_mgr).

-behaviour(gen_server).

-define(SERVER, ?MODULE).

%% API
-export([start/2, start_link/2, stop/0]).
-export([cleanup/0, dump_logs/0, setup_nodes/1, start_node/1, stop_node/2, 
         get_service_address/2, get_internal_address/2, get_hostname/1,
         get_config/2, get_log_path/1, get_log_paths/0, log/2]).

%% gen_server callbacks
-export([ init/1
        , handle_call/3
        , handle_cast/2
        , handle_info/2
        , terminate/2
        , code_change/3 ]).

%=== MACROS ====================================================================

-define(CALL_TAG, ?MODULE).
-define(CT_CONF_KEY, node_manager).
-define(CALL_TIMEOUT, 60000).
-define(NODE_TEARDOWN_TIMEOUT, 0).
-define(DEFAULT_HTTP_TIMEOUT, 3000).

%=== TYPES ====================================================================



%=== GENERIC API FUNCTIONS =====================================================

start(Backends, EnvMap) ->
    {ok, _} = application:ensure_all_started(hackney),
    gen_server:start({local, ?SERVER}, ?MODULE, [Backends, EnvMap], []).

start_link(Backends, EnvMap) ->
    {ok, _} = application:ensure_all_started(hackney),
    gen_server:start_link({local, ?SERVER}, ?MODULE, [Backends, EnvMap], []).

stop() ->
    gen_server:stop(?SERVER).

cleanup() ->
    gen_server:call(?SERVER, cleanup).

dump_logs() ->
    gen_server:call(?SERVER, dump_logs).

get_log_paths() ->
    gen_server:call(?SERVER, get_log_paths).

get_log_path(NodeName) ->
    gen_server:call(?SERVER, {get_log_path, NodeName}).

setup_nodes(NodeSpecs) ->
    gen_server:call(?SERVER, {setup_nodes, NodeSpecs}).

start_node(NodeName) ->
    gen_server:call(?SERVER, {start_node, NodeName}).

stop_node(NodeName, Timeout) ->
    gen_server:call(?SERVER, {stop_node, NodeName, Timeout}, 60000).

get_service_address(NodeName, Service) ->
    gen_server:call(?SERVER, {get_service_address, NodeName, Service}).

get_internal_address(NodeName, Service) ->
    gen_server:call(?SERVER, {get_internal_address, NodeName, Service}).

%% provide Keys as path, e.g. ["mining", "beneficiary_reward_delay"]
get_config(NodeName, Keys) ->
    gen_server:call(?SERVER, {get_config, NodeName, Keys}).

get_hostname(NodeName) ->
    gen_server:call(?SERVER, {get_hostname, NodeName}).

log(Format, Params) ->
    gen_server:call(?SERVER, {log, Format, Params}).

%=== BEHAVIOUR GEN_SERVER CALLBACK FUNCTIONS ===================================

init([Backends, EnvMap]) ->
    Opts = EnvMap#{test_id => maps:get(test_id, EnvMap, <<"quickcheck">>), 
                   log_fun => maps:get(log_fun, EnvMap, undefined)},
    InitialState = Opts,
    %% why keeping dirs twice??
    {ok, InitialState#{backends => [{Backend, Backend:start(Opts)} || Backend <- Backends ],
                       nodes => #{} }}.

handle_call(get_log_paths, _From, State) ->
    {reply, mgr_get_log_paths(State), State};
handle_call({get_log_path, NodeName}, _From, State) ->
    {reply, mgr_get_log_path(State, NodeName), State};
handle_call(cleanup, _From, State) ->
    CleanState = mgr_cleanup(State),
    {reply, ok, CleanState};
handle_call(stop, _From, State) ->
    {stop, normal, ok, State};
handle_call(dump_logs, _From, #{nodes := Nodes} = State) ->
    [ Backend:node_logs(Node) || {Backend, Node} <- maps:values(Nodes) ],
    {reply, ok, State};
handle_call({setup_nodes, NodeSpecs}, _From, State) ->
    #{backends := Backends, nodes := Nodes} = State,
    PreparedSpecs = mgr_prepare_specs(NodeSpecs, State),
    SetupNodes = [mgr_setup_node(Spec, Backends) || Spec <- PreparedSpecs],
    NewNodes = add_nodes(SetupNodes, Nodes),
    {reply, ok, State#{nodes => NewNodes}};
handle_call({get_node_pubkey, NodeName}, _From, State) ->
    {reply, mgr_get_node_pubkey(NodeName, State), State};
handle_call({get_service_address, NodeName, Service}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    ServiceAddress = Mod:get_service_address(Service, NodeState),
    {reply, ServiceAddress, State};
handle_call({get_internal_address, NodeName, Service}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    ServiceAddress = Mod:get_internal_address(Service, NodeState),
    {reply, ServiceAddress, State};
handle_call({get_config, NodeName, Path}, _From, #{nodes := Nodes} = State) when is_list(Path) ->
    #{NodeName := {_Mod, NodeState}} = Nodes,
    Config = maps:get(config, NodeState, []),
    {reply, config_find(Path, Config), State};
handle_call({get_hostname, NodeName}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {_Mod, _NodeState}} = Nodes,
    %% for the moment only localhost supported
    {reply, "localhost", State};
handle_call({start_node, NodeName}, _From, #{nodes := Nodes} = State) ->
    {Mod, NodeState} = maps:get(NodeName, Nodes),
    NodeState2 = Mod:start_node(NodeState),
    {reply, ok, State#{nodes := Nodes#{NodeName := {Mod, NodeState2}}}};
handle_call({stop_node, NodeName, Timeout}, _From, #{nodes := Nodes} =State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    Opts = #{soft_timeout => Timeout},
    NodeState2 = Mod:stop_node(NodeState, Opts),
    {reply, ok, State#{nodes := Nodes#{NodeName := {Mod, NodeState2}}}};
handle_call({kill_node, NodeName}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    NodeState2 = Mod:kill_node(NodeState),
    {reply, ok, State#{nodes := Nodes#{NodeName := {Mod, NodeState2}}}};
handle_call({extract_archive, NodeName, Path, Archive}, _From, State) ->
    {reply, ok, mgr_extract_archive(NodeName, Path, Archive, State)};
handle_call({run_cmd_in_node_dir, NodeName, Cmd, Timeout}, _From, State) ->
    {ok, Reply, NewState} = mgr_run_cmd_in_node_dir(NodeName, Cmd, Timeout, State),
    {reply, Reply, NewState};
handle_call({connect_node, NodeName, NetName}, _From, State) ->
    {reply, ok, mgr_connect_node(NodeName, NetName, State)};
handle_call({disconnect_node, NodeName, NetName}, _From, State) ->
    {reply, ok, mgr_disconnect_node(NodeName, NetName, State)};
handle_call({log, Format, Params}, _From, #{log_fun := LogFun} = State) ->
    log(LogFun, Format, Params),
    {reply, ok, State};
handle_call({export, NodeName, Name}, _From, State) ->
    {ok, Reply, NewState} = mgr_export_node(NodeName, Name, State),
    {reply, Reply, NewState};
handle_call(Request, From, _State) ->
    erlang:error({unknown_request, Request, From}).

handle_info(_Msg, State) ->
    {noreply, State}.

handle_cast(_Msg, State) ->
    {noreply, State}.

terminate(_Reason, State) ->
    mgr_cleanup(State),
    ok.

code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%=== INTERNAL FUNCTIONS ========================================================

log(LogFun, Format, Params) when is_function(LogFun) ->
    LogFun(Format, Params);
log(#{log_fun := LogFun}, Format, Params) when is_function(LogFun) ->
    LogFun(Format, Params);
log(_, _, _) -> ok.

mgr_get_log_paths(#{nodes := Nodes}) ->
    maps:fold(fun(NodeName, {Mod, NodeState}, Acc) ->
        Acc#{NodeName => Mod:get_log_path(NodeState)}
    end, #{}, Nodes).

mgr_get_log_path(#{nodes := Nodes}, NodeName) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    Mod:get_log_path(NodeState).

mgr_cleanup(State) ->
    %% Node cleanup can be disabled for debugging,
    %% then we keep dockers running
    case os:getenv("EPOCH_DISABLE_NODE_CLEANUP") of<commit_msg>Start dependencies needed by system tests<commit_after>%%% -*- erlang-indent-level: 4 -*-
%%%-------------------------------------------------------------------
%%% @copyright (C) 2017, <NAME>
%%% @doc Keeping state of running nodes during system test.
%%% @end
%%% --------------------------------------------------------------------

-module(aest_nodes_mgr).

-behaviour(gen_server).

-define(SERVER, ?MODULE).

%% API
-export([start/2, start_link/2, stop/0]).
-export([cleanup/0, dump_logs/0, setup_nodes/1, start_node/1, stop_node/2, 
         get_service_address/2, get_internal_address/2, get_hostname/1,
         get_config/2, get_log_path/1, get_log_paths/0, log/2]).

%% gen_server callbacks
-export([ init/1
        , handle_call/3
        , handle_cast/2
        , handle_info/2
        , terminate/2
        , code_change/3 ]).

%=== MACROS ====================================================================

-define(CALL_TAG, ?MODULE).
-define(CT_CONF_KEY, node_manager).
-define(CALL_TIMEOUT, 60000).
-define(NODE_TEARDOWN_TIMEOUT, 0).
-define(DEFAULT_HTTP_TIMEOUT, 3000).

%=== TYPES ====================================================================



%=== GENERIC API FUNCTIONS =====================================================

start(Backends, EnvMap) -> start_proc(?FUNCTION_NAME, Backends, EnvMap).

start_link(Backends, EnvMap) -> start_proc(?FUNCTION_NAME, Backends, EnvMap).

stop() ->
    gen_server:stop(?SERVER).

cleanup() ->
    gen_server:call(?SERVER, cleanup).

dump_logs() ->
    gen_server:call(?SERVER, dump_logs).

get_log_paths() ->
    gen_server:call(?SERVER, get_log_paths).

get_log_path(NodeName) ->
    gen_server:call(?SERVER, {get_log_path, NodeName}).

setup_nodes(NodeSpecs) ->
    gen_server:call(?SERVER, {setup_nodes, NodeSpecs}).

start_node(NodeName) ->
    gen_server:call(?SERVER, {start_node, NodeName}).

stop_node(NodeName, Timeout) ->
    gen_server:call(?SERVER, {stop_node, NodeName, Timeout}, 60000).

get_service_address(NodeName, Service) ->
    gen_server:call(?SERVER, {get_service_address, NodeName, Service}).

get_internal_address(NodeName, Service) ->
    gen_server:call(?SERVER, {get_internal_address, NodeName, Service}).

%% provide Keys as path, e.g. ["mining", "beneficiary_reward_delay"]
get_config(NodeName, Keys) ->
    gen_server:call(?SERVER, {get_config, NodeName, Keys}).

get_hostname(NodeName) ->
    gen_server:call(?SERVER, {get_hostname, NodeName}).

log(Format, Params) ->
    gen_server:call(?SERVER, {log, Format, Params}).

%=== BEHAVIOUR GEN_SERVER CALLBACK FUNCTIONS ===================================

init([Backends, EnvMap]) ->
    Opts = EnvMap#{test_id => maps:get(test_id, EnvMap, <<"quickcheck">>), 
                   log_fun => maps:get(log_fun, EnvMap, undefined)},
    InitialState = Opts,
    %% why keeping dirs twice??
    {ok, InitialState#{backends => [{Backend, Backend:start(Opts)} || Backend <- Backends ],
                       nodes => #{} }}.

handle_call(get_log_paths, _From, State) ->
    {reply, mgr_get_log_paths(State), State};
handle_call({get_log_path, NodeName}, _From, State) ->
    {reply, mgr_get_log_path(State, NodeName), State};
handle_call(cleanup, _From, State) ->
    CleanState = mgr_cleanup(State),
    {reply, ok, CleanState};
handle_call(stop, _From, State) ->
    {stop, normal, ok, State};
handle_call(dump_logs, _From, #{nodes := Nodes} = State) ->
    [ Backend:node_logs(Node) || {Backend, Node} <- maps:values(Nodes) ],
    {reply, ok, State};
handle_call({setup_nodes, NodeSpecs}, _From, State) ->
    #{backends := Backends, nodes := Nodes} = State,
    PreparedSpecs = mgr_prepare_specs(NodeSpecs, State),
    SetupNodes = [mgr_setup_node(Spec, Backends) || Spec <- PreparedSpecs],
    NewNodes = add_nodes(SetupNodes, Nodes),
    {reply, ok, State#{nodes => NewNodes}};
handle_call({get_node_pubkey, NodeName}, _From, State) ->
    {reply, mgr_get_node_pubkey(NodeName, State), State};
handle_call({get_service_address, NodeName, Service}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    ServiceAddress = Mod:get_service_address(Service, NodeState),
    {reply, ServiceAddress, State};
handle_call({get_internal_address, NodeName, Service}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    ServiceAddress = Mod:get_internal_address(Service, NodeState),
    {reply, ServiceAddress, State};
handle_call({get_config, NodeName, Path}, _From, #{nodes := Nodes} = State) when is_list(Path) ->
    #{NodeName := {_Mod, NodeState}} = Nodes,
    Config = maps:get(config, NodeState, []),
    {reply, config_find(Path, Config), State};
handle_call({get_hostname, NodeName}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {_Mod, _NodeState}} = Nodes,
    %% for the moment only localhost supported
    {reply, "localhost", State};
handle_call({start_node, NodeName}, _From, #{nodes := Nodes} = State) ->
    {Mod, NodeState} = maps:get(NodeName, Nodes),
    NodeState2 = Mod:start_node(NodeState),
    {reply, ok, State#{nodes := Nodes#{NodeName := {Mod, NodeState2}}}};
handle_call({stop_node, NodeName, Timeout}, _From, #{nodes := Nodes} =State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    Opts = #{soft_timeout => Timeout},
    NodeState2 = Mod:stop_node(NodeState, Opts),
    {reply, ok, State#{nodes := Nodes#{NodeName := {Mod, NodeState2}}}};
handle_call({kill_node, NodeName}, _From, #{nodes := Nodes} = State) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    NodeState2 = Mod:kill_node(NodeState),
    {reply, ok, State#{nodes := Nodes#{NodeName := {Mod, NodeState2}}}};
handle_call({extract_archive, NodeName, Path, Archive}, _From, State) ->
    {reply, ok, mgr_extract_archive(NodeName, Path, Archive, State)};
handle_call({run_cmd_in_node_dir, NodeName, Cmd, Timeout}, _From, State) ->
    {ok, Reply, NewState} = mgr_run_cmd_in_node_dir(NodeName, Cmd, Timeout, State),
    {reply, Reply, NewState};
handle_call({connect_node, NodeName, NetName}, _From, State) ->
    {reply, ok, mgr_connect_node(NodeName, NetName, State)};
handle_call({disconnect_node, NodeName, NetName}, _From, State) ->
    {reply, ok, mgr_disconnect_node(NodeName, NetName, State)};
handle_call({log, Format, Params}, _From, #{log_fun := LogFun} = State) ->
    log(LogFun, Format, Params),
    {reply, ok, State};
handle_call({export, NodeName, Name}, _From, State) ->
    {ok, Reply, NewState} = mgr_export_node(NodeName, Name, State),
    {reply, Reply, NewState};
handle_call(Request, From, _State) ->
    erlang:error({unknown_request, Request, From}).

handle_info(_Msg, State) ->
    {noreply, State}.

handle_cast(_Msg, State) ->
    {noreply, State}.

terminate(_Reason, State) ->
    mgr_cleanup(State),
    ok.

code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%=== INTERNAL FUNCTIONS ========================================================

start_proc(Func, Backends, EnvMap) ->
    [{ok, _} = application:ensure_all_started(A) || A <- [
        hackney,
        yamerl
    ]],
    gen_server:Func({local, ?SERVER}, ?MODULE, [Backends, EnvMap], []).

log(LogFun, Format, Params) when is_function(LogFun) ->
    LogFun(Format, Params);
log(#{log_fun := LogFun}, Format, Params) when is_function(LogFun) ->
    LogFun(Format, Params);
log(_, _, _) -> ok.

mgr_get_log_paths(#{nodes := Nodes}) ->
    maps:fold(fun(NodeName, {Mod, NodeState}, Acc) ->
        Acc#{NodeName => Mod:get_log_path(NodeState)}
    end, #{}, Nodes).

mgr_get_log_path(#{nodes := Nodes}, NodeName) ->
    #{NodeName := {Mod, NodeState}} = Nodes,
    Mod:get_log_path(NodeState).

mgr_cleanup(State) ->
    %% Node cleanup can be disabled for debugging,
    %% then we keep dockers running
    case os:getenv("EPOCH_DISABLE_NODE_CLEANUP") of
{'fraction_non_alphanumeric': 0.21231147647096899, 'fraction_numerical': 0.006084536215936307, 'mean_word_length': 4.373913043478261, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 120, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3732906', 'n_tokens_mistral': 5310, 'n_tokens_neox': 5195, 'n_words': 1065}