<commit_before>-module(simple_http_worker).

-export([initial_state/0, metrics/0,
         get/3, get/4, random_get/3]).

-type state() :: term().
-type meta() :: [{Key :: atom(), Value :: any()}].

-spec initial_state() -> state().
initial_state() -> [].

-spec metrics() -> list().
metrics() ->
    [
        {group, "Summary", [
            {graph, #{title => "HTTP Response",
                      units => "N",
                      metrics => [{"http_ok", counter}, {"http_fail", counter}, {"other_fail", counter}]}},
            {graph, #{title => "Latency",
                      units => "microseconds",
                      metrics => [{"latency", histogram}]}}
        ]}
    ].

-spec get(state(), meta(), string()) -> {nil, state()}.
get(State, _Meta, URL) ->
    get(State, _Meta, URL, []).

-spec get(state(), meta(), string(), [{atom(), term()}]) -> {nil, state()}.
get(State, _Meta, URL, Options) ->
    StartTime = os:timestamp(),
    ShouldLog = proplists:get_value(log, Options, false),
    ExpectedCode = proplists:get_value(expected, Options, 200),

    ShouldLog andalso lager:info("GET ~s", [URL]),

    {HackneyOptions, Headers} = parse_options(Options),
    Response = hackney:request(get, list_to_binary(URL), Headers, <<"">>, HackneyOptions),

    case Response of
        {ok, _, _, BodyRef} -> hackney:skip_body(BodyRef);
        _ -> ok
    end,

    Latency = timer:now_diff(os:timestamp(), StartTime),
    mzb_metrics:notify({"latency", histogram}, Latency),

    case Response of
        {ok, ExpectedCode, _, _} ->
            mzb_metrics:notify({"http_ok", counter}, 1);
        {ok, _, _, _} = Reply ->
            lager:error("GET failed: ~p", [Reply]),
            mzb_metrics:notify({"http_fail", counter}, 1);
        E ->
            lager:error("hackney:request failed: ~p", [E]),
            mzb_metrics:notify({"other_fail", counter}, 1)
    end,
    {nil, State}.

random_get(State, Meta, List) ->
    {URL, Options} = weighted_choose(List),
    get(State, Meta, URL, Options).

weighted_choose(List) ->
    Sum = lists:sum([W || {_, W} <- List]),
    R = random:uniform(Sum),
    choose(R, List).<commit_msg>Add http post in simple_http worker<commit_after>-module(simple_http_worker).

-export([initial_state/0, metrics/0,
         get/3, get/4, post/5, random_get/3]).

-type state() :: term().
-type meta() :: [{Key :: atom(), Value :: any()}].

-spec initial_state() -> state().
initial_state() -> [].

-spec metrics() -> list().
metrics() ->
    [
        {group, "Summary", [
            {graph, #{title => "HTTP Response",
                      units => "N",
                      metrics => [{"http_ok", counter}, {"http_fail", counter}, {"other_fail", counter}]}},
            {graph, #{title => "Latency",
                      units => "microseconds",
                      metrics => [{"latency", histogram}]}}
        ]}
    ].

-spec get(state(), meta(), string()) -> {nil, state()}.
get(State, _Meta, URL) ->
    get(State, _Meta, URL, []).

-spec get(state(), meta(), string(), [{atom(), term()}]) -> {nil, state()}.
get(State, _Meta, URL, Options) ->
    ShouldLog = proplists:get_value(log, Options, false),
    ExpectedCode = proplists:get_value(expected, Options, 200),

    ShouldLog andalso lager:info("GET ~s", [URL]),

    {HackneyOptions, Headers} = parse_options(Options),
    StartTime = os:timestamp(),
    Response = hackney:request(get, list_to_binary(URL), Headers, <<"">>, HackneyOptions),
    Latency = timer:now_diff(os:timestamp(), StartTime),

    case Response of
        {ok, _, _, BodyRef} -> hackney:skip_body(BodyRef);
        _ -> ok
    end,

    mzb_metrics:notify({"latency", histogram}, Latency),

    case Response of
        {ok, ExpectedCode, _, _} ->
            mzb_metrics:notify({"http_ok", counter}, 1);
        {ok, _, _, _} = Reply ->
            lager:error("GET failed: ~p", [Reply]),
            mzb_metrics:notify({"http_fail", counter}, 1);
        E ->
            lager:error("hackney:request failed: ~p", [E]),
            mzb_metrics:notify({"other_fail", counter}, 1)
    end,
    {nil, State}.

post(State, _Meta, URL, Body, Options) ->
    ShouldLog = proplists:get_value(log, Options, false),
    ExpectedCode = proplists:get_value(expected, Options, 200),

    ShouldLog andalso lager:info("POST ~s", [URL]),

    {HackneyOptions, Headers} = parse_options(Options),
    StartTime = os:timestamp(),
    Response = hackney:request(post, list_to_binary(URL), Headers, list_to_binary(Body), HackneyOptions),
    Latency = timer:now_diff(os:timestamp(), StartTime),

    case Response of
        {ok, _, _, BodyRef} -> hackney:skip_body(BodyRef);
        _ -> ok
    end,

    mzb_metrics:notify({"latency", histogram}, Latency),

    case Response of
        {ok, ExpectedCode, _, _} ->
            mzb_metrics:notify({"http_ok", counter}, 1);
        {ok, _, _, _} = Reply ->
            lager:error("POST failed: ~p", [Reply]),
            mzb_metrics:notify({"http_fail", counter}, 1);
        E ->
            lager:error("hackney:request failed: ~p", [E]),
            mzb_metrics:notify({"other_fail", counter}, 1)
    end,
    {nil, State}.

random_get(State, Meta, List) ->
    {URL, Options} = weighted_choose(List),
    get(State, Meta, URL, Options).

weighted_choose(List) ->
    Sum = lists:sum([W || {_, W} <- List]),
    R = random:uniform(Sum),
    choose(R, List).
{'fraction_non_alphanumeric': 0.18832252085264134, 'fraction_numerical': 0.005375347544022243, 'mean_word_length': 2.7707896575821103, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '406423', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1720, 'n_words': 384}