<commit_before>%% @author Couchbase, Inc <<EMAIL>>
%% @copyright 2015-2018 Couchbase, Inc.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%      http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
-module(service_stats_collector).

-include_lib("eunit/include/eunit.hrl").

-include("ns_common.hrl").

-include("ns_stats.hrl").

%% API
-export([start_link/1, service_prefix/1, service_event_name/1,
         global_stat/2, per_item_stat/3]).

%% callbacks
-export([init/1, handle_info/2, grab_stats/1, process_stats/5]).

-record(state, {service :: atom(),
                default_stats,
                buckets}).

-record(stats_accumulators, {
          gauges = [],
          counters = [],
          sys_gauges = [],
          sys_counters = [],
          status = []
         }).

server_name(Service) ->
    list_to_atom(?MODULE_STRING "-" ++ atom_to_list(Service:get_type())).

ets_name(Service) ->
    list_to_atom(?MODULE_STRING "_names-" ++ atom_to_list(Service:get_type())).

start_link(Service) ->
    base_stats_collector:start_link({local, server_name(Service)}, ?MODULE,
                                    Service).

service_prefix(Service) ->
    "@" ++ atom_to_list(Service:get_type()) ++ "-".

service_stat_prefix(Service) ->
    atom_to_list(Service:get_type()) ++ "_".

service_event_name(Service) ->
    "@" ++ atom_to_list(Service:get_type()).

per_item_stat(Service, Item, Metric) ->
    iolist_to_binary([atom_to_list(Service:get_type()), $/, Item, $/, Metric]).

global_stat(Service, StatName) ->
    iolist_to_binary([atom_to_list(Service:get_type()), $/, StatName]).

init(Service) ->
    ets:new(ets_name(Service), [protected, named_table]),

    Self = self(),
    ns_pubsub:subscribe_link(
      ns_config_events,
      fun ({buckets, Buckets}) ->
              BucketConfigs = proplists:get_value(configs, Buckets, []),
              BucketsList = ns_bucket:get_bucket_names_of_type(membase, couchstore, BucketConfigs) ++
                  ns_bucket:get_bucket_names_of_type(membase, ephemeral, BucketConfigs),
              Self ! {buckets, BucketsList};
          (_) ->
              ok
      end),

    Buckets = lists:map(fun list_to_binary/1,
                        ns_bucket:get_bucket_names_of_type(membase, couchstore) ++
                            ns_bucket:get_bucket_names_of_type(membase, ephemeral)),
    Defaults = [{global_stat(Service, atom_to_binary(Stat, latin1)), 0}
                || Stat <- Service:get_gauges() ++ Service:get_counters() ++
                       Service:get_computed()],


    {ok, #state{service = Service,
                buckets = Buckets,
                default_stats = finalize_stats(Defaults)}}.

find_type(_, []) ->
    not_found;
find_type(Name, [{Type, Metrics} | Rest]) ->
    MaybeMetric = [Name || M <- Metrics,
                           atom_to_binary(M, latin1) =:= Name],

    case MaybeMetric of
        [_] ->
            Type;
        _ ->
            find_type(Name, Rest)
    end.

global_types(Service) ->
    [{#stats_accumulators.sys_gauges, Service:get_service_gauges()},
     {#stats_accumulators.sys_counters, Service:get_service_counters()}].

bucket_types(Service) ->
    [{#stats_accumulators.gauges, Service:get_gauges()},
     {#stats_accumulators.counters, Service:get_counters()}].

do_recognize_name(_Service, <<"needs_restart">>) ->
    {#stats_accumulators.status, index_needs_restart};
do_recognize_name(_Service, <<"num_connections">>) ->
    {#stats_accumulators.status, index_num_connections};
do_recognize_name(Service, K) when is_binary(K) ->
    case find_type(K, global_types(Service)) of
        not_found ->
            do_recognize_complex_name(Service, K);
        Type ->
            NewKey = list_to_binary(service_stat_prefix(Service) ++
                                        binary_to_list(K)),
            {Type, NewKey}
    end;
do_recognize_name(Service, K) ->
    do_recognize_complex_name(Service, K).

do_recognize_complex_name(Service, K) ->
    case Service:split_stat_name(K) of
        [Bucket, Item, Metric] ->
            case find_type(Metric, bucket_types(Service)) of
                not_found ->
                    undefined;
                Type ->
                    {Type, {Bucket, Item, Metric}}
            end;
        [Item, Metric] ->
            case find_type(Metric, global_types(Service)) of
                not_found ->
                    undefined;
                Type ->
                    {Type, {Item, Metric}}
            end;
        _ ->
            undefined
    end.

recognize_name(Service, Ets, K) ->
    case ets:lookup(Ets, K) of
        [{K, Type, NewK}] ->
            {Type, NewK};
        [{K, undefined}] ->
            undefined;
        [] ->
            case do_recognize_name(Service, K) of
                undefined ->
                    ets:insert(Ets, {K, undefined}),
                    undefined;
                {Type, NewK} ->
                    ets:insert(Ets, {K, Type, NewK}),
                    {Type, NewK}
            end
    end.

massage_stats(Service, Ets, GrabbedStats) ->
    massage_stats(Service, Ets, GrabbedStats, #stats_accumulators{}).

massage_stats(_Service, _Ets, [], Acc) ->
    Acc;
massage_stats(Service, Ets, [{K, V} | Rest], Acc) ->
    case recognize_name(Service, Ets, K) of
        undefined ->
            massage_stats(Service, Ets, Rest, Acc);
        {Pos, NewK} ->
            massage_stats(
              Service, Ets, Rest,
              setelement(Pos, Acc, [{NewK, V} | element(Pos, Acc)]))
    end.

grab_stats(#state{service = Service}) ->
    case ns_cluster_membership:should_run_service(ns_config:latest(),
                                                  Service:get_type(), node()) of
        true ->
            do_grab_stats(Service);
        false ->
            []
    end.

do_grab_stats(Service) ->
    case Service:grab_stats() of
        {ok, {[_|_] = Stats}} ->
            Stats;
        {ok, Other} ->
            ?log_error("Got invalid stats response for ~p:~n~p",
                       [Service, Other]),
            [];
        {error, _} ->
            []
    end.

process_stats(TS, GrabbedStats, PrevCounters, PrevTS,
              #state{service = Service,<commit_msg>do not complain when service returns empty stats<commit_after>%% @author Couchbase, Inc <<EMAIL>>
%% @copyright 2015-2018 Couchbase, Inc.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%      http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
-module(service_stats_collector).

-include_lib("eunit/include/eunit.hrl").

-include("ns_common.hrl").

-include("ns_stats.hrl").

%% API
-export([start_link/1, service_prefix/1, service_event_name/1,
         global_stat/2, per_item_stat/3]).

%% callbacks
-export([init/1, handle_info/2, grab_stats/1, process_stats/5]).

-record(state, {service :: atom(),
                default_stats,
                buckets}).

-record(stats_accumulators, {
          gauges = [],
          counters = [],
          sys_gauges = [],
          sys_counters = [],
          status = []
         }).

server_name(Service) ->
    list_to_atom(?MODULE_STRING "-" ++ atom_to_list(Service:get_type())).

ets_name(Service) ->
    list_to_atom(?MODULE_STRING "_names-" ++ atom_to_list(Service:get_type())).

start_link(Service) ->
    base_stats_collector:start_link({local, server_name(Service)}, ?MODULE,
                                    Service).

service_prefix(Service) ->
    "@" ++ atom_to_list(Service:get_type()) ++ "-".

service_stat_prefix(Service) ->
    atom_to_list(Service:get_type()) ++ "_".

service_event_name(Service) ->
    "@" ++ atom_to_list(Service:get_type()).

per_item_stat(Service, Item, Metric) ->
    iolist_to_binary([atom_to_list(Service:get_type()), $/, Item, $/, Metric]).

global_stat(Service, StatName) ->
    iolist_to_binary([atom_to_list(Service:get_type()), $/, StatName]).

init(Service) ->
    ets:new(ets_name(Service), [protected, named_table]),

    Self = self(),
    ns_pubsub:subscribe_link(
      ns_config_events,
      fun ({buckets, Buckets}) ->
              BucketConfigs = proplists:get_value(configs, Buckets, []),
              BucketsList = ns_bucket:get_bucket_names_of_type(membase, couchstore, BucketConfigs) ++
                  ns_bucket:get_bucket_names_of_type(membase, ephemeral, BucketConfigs),
              Self ! {buckets, BucketsList};
          (_) ->
              ok
      end),

    Buckets = lists:map(fun list_to_binary/1,
                        ns_bucket:get_bucket_names_of_type(membase, couchstore) ++
                            ns_bucket:get_bucket_names_of_type(membase, ephemeral)),
    Defaults = [{global_stat(Service, atom_to_binary(Stat, latin1)), 0}
                || Stat <- Service:get_gauges() ++ Service:get_counters() ++
                       Service:get_computed()],


    {ok, #state{service = Service,
                buckets = Buckets,
                default_stats = finalize_stats(Defaults)}}.

find_type(_, []) ->
    not_found;
find_type(Name, [{Type, Metrics} | Rest]) ->
    MaybeMetric = [Name || M <- Metrics,
                           atom_to_binary(M, latin1) =:= Name],

    case MaybeMetric of
        [_] ->
            Type;
        _ ->
            find_type(Name, Rest)
    end.

global_types(Service) ->
    [{#stats_accumulators.sys_gauges, Service:get_service_gauges()},
     {#stats_accumulators.sys_counters, Service:get_service_counters()}].

bucket_types(Service) ->
    [{#stats_accumulators.gauges, Service:get_gauges()},
     {#stats_accumulators.counters, Service:get_counters()}].

do_recognize_name(_Service, <<"needs_restart">>) ->
    {#stats_accumulators.status, index_needs_restart};
do_recognize_name(_Service, <<"num_connections">>) ->
    {#stats_accumulators.status, index_num_connections};
do_recognize_name(Service, K) when is_binary(K) ->
    case find_type(K, global_types(Service)) of
        not_found ->
            do_recognize_complex_name(Service, K);
        Type ->
            NewKey = list_to_binary(service_stat_prefix(Service) ++
                                        binary_to_list(K)),
            {Type, NewKey}
    end;
do_recognize_name(Service, K) ->
    do_recognize_complex_name(Service, K).

do_recognize_complex_name(Service, K) ->
    case Service:split_stat_name(K) of
        [Bucket, Item, Metric] ->
            case find_type(Metric, bucket_types(Service)) of
                not_found ->
                    undefined;
                Type ->
                    {Type, {Bucket, Item, Metric}}
            end;
        [Item, Metric] ->
            case find_type(Metric, global_types(Service)) of
                not_found ->
                    undefined;
                Type ->
                    {Type, {Item, Metric}}
            end;
        _ ->
            undefined
    end.

recognize_name(Service, Ets, K) ->
    case ets:lookup(Ets, K) of
        [{K, Type, NewK}] ->
            {Type, NewK};
        [{K, undefined}] ->
            undefined;
        [] ->
            case do_recognize_name(Service, K) of
                undefined ->
                    ets:insert(Ets, {K, undefined}),
                    undefined;
                {Type, NewK} ->
                    ets:insert(Ets, {K, Type, NewK}),
                    {Type, NewK}
            end
    end.

massage_stats(Service, Ets, GrabbedStats) ->
    massage_stats(Service, Ets, GrabbedStats, #stats_accumulators{}).

massage_stats(_Service, _Ets, [], Acc) ->
    Acc;
massage_stats(Service, Ets, [{K, V} | Rest], Acc) ->
    case recognize_name(Service, Ets, K) of
        undefined ->
            massage_stats(Service, Ets, Rest, Acc);
        {Pos, NewK} ->
            massage_stats(
              Service, Ets, Rest,
              setelement(Pos, Acc, [{NewK, V} | element(Pos, Acc)]))
    end.

grab_stats(#state{service = Service}) ->
    case ns_cluster_membership:should_run_service(ns_config:latest(),
                                                  Service:get_type(), node()) of
        true ->
            do_grab_stats(Service);
        false ->
            []
    end.

do_grab_stats(Service) ->
    case Service:grab_stats() of
        {ok, {Stats}} when is_list(Stats) ->
            Stats;
        {ok, Other} ->
            ?log_error("Got invalid stats response for ~p:~n~p",
                       [Service, Other]),
            [];
        {error, _} ->
            []
    end.

process_stats(TS, GrabbedStats, PrevCounters, PrevTS,
              #state{service = Service,
{'fraction_non_alphanumeric': 0.13218091051337455, 'fraction_numerical': 0.0037255048059011996, 'mean_word_length': 2.27206240858118, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4309733', 'n_tokens_mistral': 4247, 'n_tokens_neox': 4098, 'n_words': 798}