<commit_before>%% @author Northscale <<EMAIL>>
%% @copyright 2009 NorthScale, Inc.
%% All rights reserved.

%% @doc Web server for menelaus.

-module(menelaus_stats).
-author('NorthScale <<EMAIL>>').

-include_lib("eunit/include/eunit.hrl").

-ifdef(EUNIT).
-export([test/0]).
-import(menelaus_util,
        [test_under_debugger/0, debugger_apply/2,
         wrap_tests_with_cache_setup/1]).
-endif.

-export([handle_bucket_stats/3, basic_stats/2]).

-import(menelaus_util,
        [reply_json/2,
         expect_prop_value/2,
         java_date/0,
         string_hash/1,
         my_seed/1,
         stateful_map/3,
         stateful_takewhile/3,
         low_pass_filter/2,
         caching_result/2]).

%% External API

basic_stats(PoolId, BucketId) ->
    Pool = menelaus_web:find_pool_by_id(PoolId),
    Bucket = menelaus_web:find_bucket_by_id(Pool, BucketId),
    MbPerNode = expect_prop_value(size_per_node, Bucket),
    NumNodes = length(ns_node_disco:nodes_wanted()),
    SamplesNum = 10,
    Samples = get_stats_raw(PoolId, BucketId, SamplesNum),
    OpsPerSec = avg(deltas(sum_stats_ops(Samples))),
    EvictionsPerSec = avg(deltas(proplists:get_value("evictions", Samples))),
    CurBytes = erlang:max(avg(proplists:get_value("bytes", Samples)),
                          1),
    MaxBytes = erlang:max(avg(proplists:get_value("limit_maxbytes", Samples)),
                          CurBytes),
    [{cacheSize, NumNodes * MbPerNode},
     {opsPerSec, OpsPerSec},
     {evictionsPerSec, EvictionsPerSec},
     {cachePercentUsed, float_round(CurBytes / MaxBytes)}].

% GET /pools/default/stats?stat=opsbysecond
% GET /pools/default/stats?stat=hot_keys

handle_bucket_stats(PoolId, all, Req) ->
    handle_bucket_stats(PoolId, "default", Req);

handle_bucket_stats(PoolId, Id, Req) ->
    handle_buckets_stats(PoolId, [Id], Req).

handle_buckets_stats(PoolId, BucketIds, Req) ->
    Params = Req:parse_qs(),
    case proplists:get_value("stat", Params) of
        "opsbysecond" ->
            handle_buckets_stats_ops(Req, PoolId, BucketIds, Params);
        "hot_keys" ->
            handle_buckets_stats_hks(Req, PoolId, BucketIds, Params);
        _ ->
            Req:respond({400, [], []})
    end.

handle_buckets_stats_ops(Req, PoolId, BucketIds, Params) ->
    Res = build_buckets_stats_ops_response(PoolId, BucketIds, Params),
    reply_json(Req, Res).

handle_buckets_stats_hks(Req, PoolId, BucketIds, Params) ->
    Res = build_buckets_stats_hks_response(PoolId, BucketIds, Params),
    reply_json(Req, Res).

%% ops SUM(cmd_get, cmd_set,
%%         incr_misses, incr_hits,
%%         decr_misses, decr_hits,
%%         cas_misses, cas_hits, cas_badval,
%%         delete_misses, delete_hits,
%%         cmd_flush)
%% cmd_get (cmd_get)
%% get_misses (get_misses)
%% get_hits (get_hits)
%% cmd_set (cmd_set)
%% evictions (evictions)
%% replacements (if available in time)
%% misses SUM(get_misses, delete_misses, incr_misses, decr_misses,
%%            cas_misses)
%% updates SUM(cmd_set, incr_hits, decr_hits, cas_hits)
%% bytes_read (bytes_read)
%% bytes_written (bytes_written)
%% hit_ratio (get_hits / cmd_get)
%% curr_items (curr_items)

%% Implementation

build_buckets_stats_ops_response(PoolId, BucketIds, Params) ->
    {ok, SamplesInterval, LastSampleTStamp, Samples2} =
        get_buckets_stats(PoolId, BucketIds, Params),
    {struct, [{op, {struct, [{tstamp, LastSampleTStamp},
                             {samplesInterval, SamplesInterval}
                             | Samples2]}}]}.

build_buckets_stats_hks_response(_PoolId, _BucketIds, _Params) ->
    {struct, [{hot_keys, [{struct, [{name, <<"product:324:inventory">>},
                                    {gets, 10000},
                                    {bucket, <<"shopping application">>},
                                    {misses, 100}]},
                          {struct, [{name, <<"user:image:value2">>},
                                    {gets, 10000},
                                    {bucket, <<"chat application">>},
                                    {misses, 100}]},
                          {struct, [{name, <<"blog:117">>},
                                    {gets, 10000},
                                    {bucket, <<"blog application">>},
                                    {misses, 100}]},
                          {struct, [{name, <<"user:image:value4">>},
                                    {gets, 10000},
                                    {bucket, <<"chat application">>},
                                    {misses, 100}]}]}]}.

get_buckets_stats(PoolId, BucketIds, Params) ->
    [FirstStats | RestStats] =
        lists:map(fun(BucketId) ->
                          get_stats(PoolId, BucketId, Params)
                  end,
                  BucketIds),
    lists:foldl(fun({ok, XSamplesInterval, XLastSampleTStamp, XStat},
                    {ok, YSamplesInterval, YLastSampleTStamp, _YStat}) ->
                        {ok,
                         erlang:max(XSamplesInterval,
                                    YSamplesInterval),
                         erlang:max(XLastSampleTStamp,
                                    YLastSampleTStamp),
                         XStat}
                end,
                FirstStats,
                RestStats).

get_stats(PoolId, BucketId, _Params) ->
    SamplesInterval = 1, % A sample every second.
    SamplesNum = 60, % Sixty seconds worth of data.
    Samples = get_stats_raw(PoolId, BucketId, SamplesNum),
    Samples2 = case lists:keytake(t, 1, Samples) of
                   false -> [{t, []} | Samples];
                   {value, {t, TStamps}, SamplesNoTStamps} -><commit_msg>opsBySecond stats for all buckets<commit_after>%% @author Northscale <<EMAIL>>
%% @copyright 2009 NorthScale, Inc.
%% All rights reserved.

%% @doc Web server for menelaus.

-module(menelaus_stats).
-author('NorthScale <<EMAIL>>').

-include_lib("eunit/include/eunit.hrl").

-ifdef(EUNIT).
-export([test/0]).
-import(menelaus_util,
        [test_under_debugger/0, debugger_apply/2,
         wrap_tests_with_cache_setup/1]).
-endif.

-export([handle_bucket_stats/3, basic_stats/2]).

-import(menelaus_util,
        [reply_json/2,
         expect_prop_value/2,
         java_date/0,
         string_hash/1,
         my_seed/1,
         stateful_map/3,
         stateful_takewhile/3,
         low_pass_filter/2,
         caching_result/2]).

%% External API

basic_stats(PoolId, BucketId) ->
    Pool = menelaus_web:find_pool_by_id(PoolId),
    Bucket = menelaus_web:find_bucket_by_id(Pool, BucketId),
    MbPerNode = expect_prop_value(size_per_node, Bucket),
    NumNodes = length(ns_node_disco:nodes_wanted()),
    SamplesNum = 10,
    Samples = get_stats_raw(PoolId, BucketId, SamplesNum),
    OpsPerSec = avg(deltas(sum_stats_ops(Samples))),
    EvictionsPerSec = avg(deltas(proplists:get_value("evictions", Samples))),
    CurBytes = erlang:max(avg(proplists:get_value("bytes", Samples)),
                          1),
    MaxBytes = erlang:max(avg(proplists:get_value("limit_maxbytes", Samples)),
                          CurBytes),
    [{cacheSize, NumNodes * MbPerNode},
     {opsPerSec, OpsPerSec},
     {evictionsPerSec, EvictionsPerSec},
     {cachePercentUsed, float_round(CurBytes / MaxBytes)}].

% GET /pools/default/stats?stat=opsbysecond
% GET /pools/default/stats?stat=hot_keys

handle_bucket_stats(PoolId, all, Req) ->
    handle_bucket_stats(PoolId, "default", Req);

handle_bucket_stats(PoolId, Id, Req) ->
    handle_buckets_stats(PoolId, [Id], Req).

handle_buckets_stats(PoolId, BucketIds, Req) ->
    Params = Req:parse_qs(),
    case proplists:get_value("stat", Params) of
        "opsbysecond" ->
            handle_buckets_stats_ops(Req, PoolId, BucketIds, Params);
        "hot_keys" ->
            handle_buckets_stats_hks(Req, PoolId, BucketIds, Params);
        _ ->
            Req:respond({400, [], []})
    end.

handle_buckets_stats_ops(Req, PoolId, BucketIds, Params) ->
    Res = build_buckets_stats_ops_response(PoolId, BucketIds, Params),
    reply_json(Req, Res).

handle_buckets_stats_hks(Req, PoolId, BucketIds, Params) ->
    Res = build_buckets_stats_hks_response(PoolId, BucketIds, Params),
    reply_json(Req, Res).

%% ops SUM(cmd_get, cmd_set,
%%         incr_misses, incr_hits,
%%         decr_misses, decr_hits,
%%         cas_misses, cas_hits, cas_badval,
%%         delete_misses, delete_hits,
%%         cmd_flush)
%% cmd_get (cmd_get)
%% get_misses (get_misses)
%% get_hits (get_hits)
%% cmd_set (cmd_set)
%% evictions (evictions)
%% replacements (if available in time)
%% misses SUM(get_misses, delete_misses, incr_misses, decr_misses,
%%            cas_misses)
%% updates SUM(cmd_set, incr_hits, decr_hits, cas_hits)
%% bytes_read (bytes_read)
%% bytes_written (bytes_written)
%% hit_ratio (get_hits / cmd_get)
%% curr_items (curr_items)

%% Implementation

build_buckets_stats_ops_response(PoolId, BucketIds, Params) ->
    {ok, SamplesInterval, LastSampleTStamp, Samples2} =
        get_buckets_stats(PoolId, BucketIds, Params),
    {struct, [{op, {struct, [{tstamp, LastSampleTStamp},
                             {samplesInterval, SamplesInterval}
                             | Samples2]}}]}.

build_buckets_stats_hks_response(_PoolId, _BucketIds, _Params) ->
    % TODO: hot key stats.
    {struct, [{hot_keys, [{struct, [{name, <<"product:324:inventory">>},
                                    {gets, 10000},
                                    {bucket, <<"shopping application">>},
                                    {misses, 100}]},
                          {struct, [{name, <<"user:image:value2">>},
                                    {gets, 10000},
                                    {bucket, <<"chat application">>},
                                    {misses, 100}]},
                          {struct, [{name, <<"blog:117">>},
                                    {gets, 10000},
                                    {bucket, <<"blog application">>},
                                    {misses, 100}]},
                          {struct, [{name, <<"user:image:value4">>},
                                    {gets, 10000},
                                    {bucket, <<"chat application">>},
                                    {misses, 100}]}]}]}.

get_buckets_stats(PoolId, BucketIds, Params) ->
    [FirstStats | RestStats] =
        lists:map(fun(BucketId) ->
                          get_stats(PoolId, BucketId, Params)
                  end,
                  BucketIds),
    lists:foldl(
      fun({ok, XSamplesInterval, XLastSampleTStamp, XStat},
          {ok, YSamplesInterval, YLastSampleTStamp, YStat}) ->
              {ok,
               erlang:max(XSamplesInterval,
                          YSamplesInterval),
               erlang:max(XLastSampleTStamp,
                          YLastSampleTStamp),
               lists:map(fun({t, TStamps}) -> {t, TStamps};
                            ({Key, XVals}) ->
                                 case proplists:get_value(Key, YStat) of
                                     undefined -> {Key, XVals};
                                     YVals ->
                                         {Key, lists:zipwith(
                                                 fun(A, B) -> A + B end,
                                                 XVals, YVals)}
                                 end
                        end,
                        XStat)}
      end,
      FirstStats,
      RestStats).

get_stats(PoolId, BucketId, _Params) ->
    SamplesInterval = 1, % A sample every second.
    SamplesNum = 60, % Sixty seconds worth of data.
    Samples = get_stats_raw(PoolId, BucketId, SamplesNum),
    Samples2 = case lists:keytake(t, 1, Samples) of
                   false -> [{t, []} | Samples];
                   {value, {t, TStamps}, SamplesNoTStamps} ->
{'fraction_non_alphanumeric': 0.12507379606983215, 'fraction_numerical': 0.012144724635236569, 'mean_word_length': 2.0412926391382404, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '200620', 'n_tokens_mistral': 3964, 'n_tokens_neox': 3716, 'n_words': 658}