<commit_before>%%%-------------------------------------------------------------------
%%% File:      web_rpc.erl
%%% @author    <NAME> <<EMAIL>> []
%%% @copyright 2008 <NAME>
%%% @doc  
%%%
%%% @end  
%%%
%%% @since 2008-08-14 by <NAME>
%%%-------------------------------------------------------------------
-module(web_rpc).
-author('<EMAIL>').

%% API
-export([info/1, rates/1, syncs_running/1]).

%%====================================================================
%% API
%%====================================================================
%%--------------------------------------------------------------------
%% @spec 
%% @doc
%% @end 
%%--------------------------------------------------------------------

info(stats) ->
  {obj, [
    {node,node()}, 
    {running_nodes,lists:sort(nodes([this,visible]))},
    {member_nodes,transform_partitions(lists:keysort(1, membership:partitions()))}
  ]}.

rates(cluster) ->
  {obj, [
    {get_rate, lists:foldl(fun(Node, Acc) -> Acc + socket_server:rate(Node, get_rate, 1) end, 0, nodes([this,visible]))},
    {put_rate, lists:foldl(fun(Node, Acc) -> Acc + socket_server:rate(Node, put_rate, 1) end, 0, nodes([this,visible]))},
    {in_rate, lists:foldl(fun(Node, Acc) -> Acc + socket_server:rate(Node, in_rate, 1) end, 0, nodes([this,visible]))},
    {out_rate, lists:foldl(fun(Node, Acc) -> Acc + socket_server:rate(Node, out_rate, 1) end, 0, nodes([this,visible]))},
    {connections, lists:foldl(fun(Node, Acc) -> Acc + socket_server:connections(Node) end, 0, nodes([this,visible]))}
  ]};

rates(Node) ->
  {obj, [
    {get_rate, socket_server:rate(Node, get_rate, 1)},
    {put_rate, socket_server:rate(Node, put_rate, 1)},
    {in_rate, socket_server:rate(Node, in_rate, 1)},
    {out_rate, socket_server:rate(Node, out_rate, 1)},
    {connections, socket_server:connections(Node)}
  ]}.
  
syncs_running(Node) ->
  {obj, [
    {running, lists:map(fun({Part, NodeA, NodeB}) -> 
        {obj, [{partition, Part}, {nodes, [NodeA, NodeB]}]}
      end, sync_manager:running(Node))}
  ]}.

%%====================================================================
%% Internal functions
%%====================================================================

transform_partitions(Partitions) ->
  lists:map(fun([Node,Parts]) -> 
      {obj, [
        {name, Node},
        {partitions, Parts},
        {replicas, membership:replica_nodes(Node)}
      ]}
    end, transform_partitions([], Partitions)).

transform_partitions([], [{Node,Part}|Parts]) ->
  transform_partitions([[Node,[Part]]], Parts);
  
transform_partitions(NodeParts, []) ->
  [[Node, lists:sort(Parts)] || [Node, Parts] <- lists:sort(NodeParts)]; 
  
transform_partitions([[Node,NodeParts]|Others], [{Node,Part}|Parts]) ->
  transform_partitions([[Node,[Part|NodeParts]]|Others], Parts);
  
transform_partitions(Others, [{Node,Part}|Parts]) ->
  transform_partitions([[Node,[Part]]|Others], Parts).<commit_msg>cleaned up data services for viz<commit_after>%%%-------------------------------------------------------------------
%%% File:      web_rpc.erl
%%% @author    <NAME> <<EMAIL>> []
%%% @copyright 2008 <NAME>
%%% @doc  
%%%
%%% @end  
%%%
%%% @since 2008-08-14 by <NAME>
%%%-------------------------------------------------------------------
-module(web_rpc).
-author('<EMAIL>').

%% API
-export([info/1, rates/1, syncs_running/1, diff_size/1]).

%%====================================================================
%% API
%%====================================================================
%%--------------------------------------------------------------------
%% @spec 
%% @doc
%% @end 
%%--------------------------------------------------------------------

info(stats) ->
  {obj, [
    {node,node()}, 
    {running_nodes,lists:sort(nodes([this,visible]))},
    {member_nodes,transform_partitions(lists:keysort(1, membership:partitions()))}
  ]};
  
info(partitions) ->
  lists:map(fun({Node, Part}) -> 
      {obj, [{node,Node},{partition,Part}]}
    end,   lists:keysort(2, membership:partitions()));
    
info(nodes) ->
  membership:nodes().

rates(cluster) ->
  {obj, [
    {get_rate, lists:foldl(fun(Node, Acc) -> Acc + stats_server:rate(Node, get_rate, 1) end, 0, nodes([this,visible]))},
    {put_rate, lists:foldl(fun(Node, Acc) -> Acc + stats_server:rate(Node, put_rate, 1) end, 0, nodes([this,visible]))},
    {in_rate, lists:foldl(fun(Node, Acc) -> Acc + stats_server:rate(Node, in_rate, 1) end, 0, nodes([this,visible]))},
    {out_rate, lists:foldl(fun(Node, Acc) -> Acc + stats_server:rate(Node, out_rate, 1) end, 0, nodes([this,visible]))},
    {connections, lists:foldl(fun(Node, Acc) -> Acc + socket_server:connections(Node) end, 0, nodes([this,visible]))}
  ]};

rates(nodes) ->
  lists:map(fun(Node) -> rates(Node) end, membership:nodes());

rates(Node) ->
  {obj, [
    {name, Node},
    {get_rate, stats_server:rate(Node, get_rate, 1)},
    {put_rate, stats_server:rate(Node, put_rate, 1)},
    {in_rate, stats_server:rate(Node, in_rate, 1)},
    {out_rate, stats_server:rate(Node, out_rate, 1)},
    {connections, socket_server:connections(Node)}
  ]}.
  
syncs_running(cluster) ->
  {Good,_} = rpc:multicall(sync_manager, running, []),
  lists:map(fun({Part, NodeA, NodeB}) -> 
      {obj, [{partition, Part}, {nodes, [NodeA, NodeB]}]}
    end, lists:flatten(Good));
  
syncs_running(Node) ->
  lists:map(fun({Part, NodeA, NodeB}) -> 
      {obj, [{partition, Part}, {nodes, [NodeA, NodeB]}]}
    end, sync_manager:running(Node)).

diff_size(cluster) ->
  {Good,_} = rpc:multicall(sync_manager, diffs, []),
  {obj,
      lists:map(fun({Part, Diffs}) ->
          {integer_to_list(Part), {obj, Diffs}}
        end, lists:flatten(Good))
    }.

%%====================================================================
%% Internal functions
%%====================================================================

transform_partitions(Partitions) ->
  lists:map(fun([Node,Parts]) -> 
      {obj, [
        {name, Node},
        {partitions, Parts},
        {replicas, membership:replica_nodes(Node)}
      ]}
    end, transform_partitions([], Partitions)).

transform_partitions([], [{Node,Part}|Parts]) ->
  transform_partitions([[Node,[Part]]], Parts);
  
transform_partitions(NodeParts, []) ->
  [[Node, lists:sort(Parts)] || [Node, Parts] <- lists:sort(NodeParts)]; 
  
transform_partitions([[Node,NodeParts]|Others], [{Node,Part}|Parts]) ->
  transform_partitions([[Node,[Part|NodeParts]]|Others], Parts);
  
transform_partitions(Others, [{Node,Part}|Parts]) ->
  transform_partitions([[Node,[Part]]|Others], Parts).
{'fraction_non_alphanumeric': 0.34388314059646985, 'fraction_numerical': 0.009129640900791236, 'mean_word_length': 3.3674418604651164, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6370955', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2043, 'n_words': 341}