<commit_before>-module(mzb_api_bench).

-behaviour(mzb_pipeline).

-export([
    start_link/2,
    interrupt_bench/1,
    get_status/1,
    change_env/2,
    seconds/0,
    send_email_report/2,
    request_report/2,
    log_file/1,
    log_user_file/1,
    metrics_file/2,
    remote_path/2
]).


-ifdef(TEST).
-export([allocate_hosts/2]).
-endif.

-define(DEFLATE_FLUSH_INTERVAL, 10000).
-define(DIRECTOR_CALL_TIMEOUT, 30000).
-define(HOOKS_TIMEOUT, 60000).

%% mzb_pipeline callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, workflow_config/1, handle_stage/3,
         handle_pipeline_status/2]).

start_link(Id, Params) ->
    mzb_pipeline:start_link(?MODULE, [Id, Params], []).

get_status(Pid) ->
    mzb_pipeline:call(Pid, status).

change_env(Pid, Env) ->
    case mzb_pipeline:call(Pid, {change_env, Env}, 30000) of
        ok -> ok;
        {error, Reason} -> erlang:error(Reason)
    end.

interrupt_bench(Pid) ->
    mzb_pipeline:stop(Pid).

request_report(Pid, Emails) ->
    mzb_pipeline:call(Pid, {request_report, Emails}).

init([Id, Params]) ->
    Now = os:timestamp(),
    StartTime = seconds(Now),
    % Acceptance tests assume the following purpose format: bench-<id>-<timestamp>
    % {{_, M, D}, {H, Mi, S}} = calendar:now_to_universal_time(now()),
    % Purpose = mzb_string:format("bench-~2.10.0B-~2.10.0B-~2.10.0B-~2.10.0B-~2.10.0B-~b", [M,D,H,Mi,S,Id]),
    Purpose = mzb_string:format("bench-~b-~b", [Id, StartTime]),
    Includes = mzb_bc:maps_get(includes, Params, []),
    VMArgs = case maps:find(vm_args, Params) of
        {ok, [_|_] = List} -> List;
        _ -> application:get_env(mzbench_api, vm_args, undefined)
    end,
    #{name := ScriptName} = maps:get(script, Params),
    NodeInstallSpec = extract_node_install_spec(Params),
    BenchName =
        case maps:find(benchmark_name, Params) of
            {ok, undefined} -> ScriptName;
            {ok, BN} -> BN;
            error -> ScriptName
        end,
    Config = #{
        id => Id,
        benchmark_name => BenchName,
        nodes_arg => maps:get(nodes, Params),
        script => generate_script_filename(maps:get(script, Params)),
        purpose => Purpose,
        node_install_spec => NodeInstallSpec,
        env => mzbl_script:normalize_env(generate_bench_env(Params)),
        deallocate_after_bench => maps:get(deallocate_after_bench, Params),
        provision_nodes => maps:get(provision_nodes, Params),
        exclusive_node_usage => maps:get(exclusive_node_usage, Params),
        req_host => maps:get(req_host, Params),
        initial_user => maps:get(user, Params),
        director_host => undefined,
        worker_hosts => [],
        emulate_bench_crash => maps:get(emulate_bench_crash, Params),
        log_file => get_env(bench_log_file),
        log_user_file => get_env(bench_log_user_file),
        metrics_file => get_env(bench_metrics_file),
        log_compression => application:get_env(mzbench_api, bench_log_compression, undefined),
        metrics_compression => application:get_env(mzbench_api, bench_metrics_compression, undefined),
        vm_args => VMArgs,
        cloud => mzb_bc:maps_get(cloud, Params, undefined),
        node_log_port => application:get_env(mzbench_api, node_log_port, undefined),
        node_log_user_port => application:get_env(mzbench_api, node_log_user_port, undefined),
        metric_update_interval_ms => extract_metric_update_interval(Params),
        node_management_port => application:get_env(mzbench_api, node_management_port, undefined)
    },
    Data = #{
        includes => Includes
    },

    ok = init_data_dir(Config),
    InputBin = erlang:term_to_binary(Params),
    ok = file:write_file(local_path("params.bin", Config), InputBin),
    erlang:process_flag(trap_exit, true),

    LogFile = log_file(Config),
    LogHandler = get_file_writer(LogFile, maps:get(log_compression, Config)),

    LogUserFile = log_user_file(Config),
    LogUserHandler = get_file_writer(LogUserFile, maps:get(log_compression, Config)),

    State = #{
        id => Id,
        start_time => StartTime,
        finish_time => undefined,<commit_msg>Fix undefined cloud when starting from CLI<commit_after>-module(mzb_api_bench).

-behaviour(mzb_pipeline).

-export([
    start_link/2,
    interrupt_bench/1,
    get_status/1,
    change_env/2,
    seconds/0,
    send_email_report/2,
    request_report/2,
    log_file/1,
    log_user_file/1,
    metrics_file/2,
    remote_path/2
]).


-ifdef(TEST).
-export([allocate_hosts/2]).
-endif.

-define(DEFLATE_FLUSH_INTERVAL, 10000).
-define(DIRECTOR_CALL_TIMEOUT, 30000).
-define(HOOKS_TIMEOUT, 60000).

%% mzb_pipeline callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, workflow_config/1, handle_stage/3,
         handle_pipeline_status/2]).

start_link(Id, Params) ->
    mzb_pipeline:start_link(?MODULE, [Id, Params], []).

get_status(Pid) ->
    mzb_pipeline:call(Pid, status).

change_env(Pid, Env) ->
    case mzb_pipeline:call(Pid, {change_env, Env}, 30000) of
        ok -> ok;
        {error, Reason} -> erlang:error(Reason)
    end.

interrupt_bench(Pid) ->
    mzb_pipeline:stop(Pid).

request_report(Pid, Emails) ->
    mzb_pipeline:call(Pid, {request_report, Emails}).

init([Id, Params]) ->
    Now = os:timestamp(),
    StartTime = seconds(Now),
    % Acceptance tests assume the following purpose format: bench-<id>-<timestamp>
    % {{_, M, D}, {H, Mi, S}} = calendar:now_to_universal_time(now()),
    % Purpose = mzb_string:format("bench-~2.10.0B-~2.10.0B-~2.10.0B-~2.10.0B-~2.10.0B-~b", [M,D,H,Mi,S,Id]),
    Purpose = mzb_string:format("bench-~b-~b", [Id, StartTime]),
    Includes = mzb_bc:maps_get(includes, Params, []),
    VMArgs = case maps:find(vm_args, Params) of
        {ok, [_|_] = List} -> List;
        _ -> application:get_env(mzbench_api, vm_args, undefined)
    end,
    #{name := ScriptName} = maps:get(script, Params),
    NodeInstallSpec = extract_node_install_spec(Params),
    BenchName =
        case maps:find(benchmark_name, Params) of
            {ok, undefined} -> ScriptName;
            {ok, BN} -> BN;
            error -> ScriptName
        end,
    Cloud = case mzb_bc:maps_get(cloud, Params, undefined) of
                undefined -> [DefaultCloud | _] = mzb_api_cloud:list_clouds(),
                             DefaultCloud;
                Cl -> Cl
            end,
    Config = #{
        id => Id,
        benchmark_name => BenchName,
        nodes_arg => maps:get(nodes, Params),
        script => generate_script_filename(maps:get(script, Params)),
        purpose => Purpose,
        node_install_spec => NodeInstallSpec,
        env => mzbl_script:normalize_env(generate_bench_env(Params)),
        deallocate_after_bench => maps:get(deallocate_after_bench, Params),
        provision_nodes => maps:get(provision_nodes, Params),
        exclusive_node_usage => maps:get(exclusive_node_usage, Params),
        req_host => maps:get(req_host, Params),
        initial_user => maps:get(user, Params),
        director_host => undefined,
        worker_hosts => [],
        emulate_bench_crash => maps:get(emulate_bench_crash, Params),
        log_file => get_env(bench_log_file),
        log_user_file => get_env(bench_log_user_file),
        metrics_file => get_env(bench_metrics_file),
        log_compression => application:get_env(mzbench_api, bench_log_compression, undefined),
        metrics_compression => application:get_env(mzbench_api, bench_metrics_compression, undefined),
        vm_args => VMArgs,
        cloud => Cloud,
        node_log_port => application:get_env(mzbench_api, node_log_port, undefined),
        node_log_user_port => application:get_env(mzbench_api, node_log_user_port, undefined),
        metric_update_interval_ms => extract_metric_update_interval(Params),
        node_management_port => application:get_env(mzbench_api, node_management_port, undefined)
    },
    Data = #{
        includes => Includes
    },

    ok = init_data_dir(Config),
    InputBin = erlang:term_to_binary(Params),
    ok = file:write_file(local_path("params.bin", Config), InputBin),
    erlang:process_flag(trap_exit, true),

    LogFile = log_file(Config),
    LogHandler = get_file_writer(LogFile, maps:get(log_compression, Config)),

    LogUserFile = log_user_file(Config),
    LogUserHandler = get_file_writer(LogUserFile, maps:get(log_compression, Config)),

    State = #{
        id => Id,
        start_time => StartTime,
        finish_time => undefined,
{'fraction_non_alphanumeric': 0.1320754716981132, 'fraction_numerical': 0.01406304933786476, 'mean_word_length': 3.424053913945049, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3369267', 'n_tokens_mistral': 3008, 'n_tokens_neox': 2936, 'n_words': 472}