<commit_before>%% -*- mode: erlang; erlang-indent-level: 2 -*-
%%% Created : 21 Aug 2014 by  <<EMAIL>>

%% @doc
%% {[Allocator,Instance,Group,Type,Info],Count}
%%
%% Allocator         -> atom(AllocatorName++"_alloc")
%% AllocatorName     -> temp| std| sl| ll| fix| ets| eheap| driver| binary
%% Instance          -> integer()
%% Group             -> CarrierGroup | CallGroup
%% CarrierGroup      -> sbcs | mbcs (multiblockcarriers | singleblockcarriers)
%% CallGroup         -> calls
%% Type              -> CarrierType | CallType
%% CarrierType       -> sys | mseg | pool | blocks
%% CallType          -> AllocatorSpecific | sys | mseg
%% AllocatorSpecific -> AllocatorName
%% Info              -> CallInfo | CarrierInfo
%% CallInfo          -> alloc | realloc | dealloc | free
%% CarrierInfo       -> count | size
%% Count             -> size in bytes | counts
%% @end

-module('atop').
-author('').
-export([help/0]).
-export([getallocdata/0]).
-export([calls/0]).
-export([allocs/0,allocs/1,allocs/2]).
-export([aggregate/0,aggregate/1,aggregate/2]).

help() ->
  {indices(),columns()}.

getallocdata() ->
  [{[A,N,K]++gettag(K,W),getvalue(W)} ||
    A <- erlang:system_info(alloc_util_allocators),
    {instance,N,Info} <- erlang:system_info({allocator,A}),
    {K,V} <- Info,
    K=/=options,
    W <- V,
    gettag(K,W)=/=drop].

aggregate() ->
  aggregate(indices()--[instance]).
aggregate(AggregateIndices) ->
  aggregate(AggregateIndices,[]).
aggregate(AggregateIndices,FilterTags) ->
  aggr(
    AggregateIndices,
    filter(
      FilterTags,
      getallocdata())).

calls() ->
  io:fwrite("~-20w~11w~11w~11w~11w~n",[allocator,alloc,dealloc,free,realloc]),
  lists:foreach(
    fun(L)->io:fwrite("~-20w~11w~11w~11w~11w~n",L)end,
    lists:foldl(
      fun({[X,alloc]  ,A},             T)  -> [[X,A,0,0,0]|T];
         ({[X,dealloc],D},[[X,A,0,0,0]|T]) -> [[X,A,D,0,0]|T];
         ({[X,free]   ,F},[[X,A,D,0,0]|T]) -> [[X,A,D,F,0]|T];
         ({[X,realloc],R},[[X,A,D,F,0]|T]) -> [[X,A,D,F,R]|T]
      end,
      [],
      lists:sort(atop:aggregate([allocator,info],[calls])))).

allocs() ->
  allocs(allocd).
allocs(SortCol) ->
  allocs(SortCol,[allocator,group]).
allocs(SortCol,AggregateIndices) ->
  printh(AggregateIndices,columns()),
  lists:foreach(
    fun print/1,
    lists:reverse(
      lists:keysort(
        column2index(SortCol),
        lists:foldl(
          fun present/2,
          [],
          aggregate(AggregateIndices++[type,info],[size]))))).

printh(AggInds,Cols) ->
  io:fwrite("~-33w~8w~8w~8w~n",[AggInds]++tl(Cols)).
print({K,T,U,F}) ->
  io:fwrite("~-33w~8.2f~8.2f~8.2f~n",[K,T,U,F]).

present([],A) -> A;
present({K,V},A) ->
  case lists:reverse(K) of
    [size,pool|NK]   -> [{NK,V}|A];
    [size,sys|NK]    -> [{NK,V}|A];
    [size,mseg|NK]   -> [{NK,V0}|A0] = A,
                        [{NK,V+V0}|A0];
    [size,blocks|NK] -> [{NK,V0}|A0] = A,
                          [{lists:reverse(NK),
                            V0/1024/1024,
                            V/1024/1024,
                            divide(V, V0)}|A0];
    _                -> A
  end.

divide(_V,0) -> 1.0;
divide(V,V0) -> V/V0.

filter([],PL) -> PL;
filter(Tags,PL) ->
  lists:filter(
    fun({K,_})->
        lists:member(true,[lists:member(T,K) || T <- Tags])
    end,
    PL).

aggr(IndexTags,List) ->
  Inds = [tag2index(T) || T <- IndexTags],
  lists:foldl(mkaggfun(Inds),[],lists:sort(mksortfun(Inds),List)).

columns() -> [what,allocd,used,frac].
indices() -> [allocator,instance,group,type,info].

column2index(X) -> x2index(X,columns()).
tag2index(X) -> x2index(X,indices()).
<commit_msg>refactored presenttion code<commit_after>%% -*- mode: erlang; erlang-indent-level: 2 -*-
%%% Created : 21 Aug 2014 by  <<EMAIL>>

%% @doc
%% {[Allocator,Instance,Group,Type,Info],Count}
%%
%% Allocator         -> atom(AllocatorName++"_alloc")
%% AllocatorName     -> temp| std| sl| ll| fix| ets| eheap| driver| binary
%% Instance          -> integer()
%% Group             -> CarrierGroup | CallGroup
%% CarrierGroup      -> sbcs | mbcs (multiblockcarriers | singleblockcarriers)
%% CallGroup         -> calls
%% Type              -> CarrierType | CallType
%% CarrierType       -> sys | mseg | pool | blocks
%% CallType          -> AllocatorSpecific | sys | mseg
%% AllocatorSpecific -> AllocatorName
%% Info              -> CallInfo | CarrierInfo
%% CallInfo          -> alloc | realloc | dealloc | free
%% CarrierInfo       -> count | size
%% Count             -> size in bytes | counts
%% @end

-module('atop').
-author('').
-export([help/0]).
-export([getallocdata/0]).
-export([calls/0]).
-export([allocs/0,allocs/1,allocs/2]).
-export([aggregate/0,aggregate/1,aggregate/2]).

help() ->
  {indices(),columns()}.

getallocdata() ->
  [{[A,N,K]++gettag(K,W),getvalue(W)} ||
    A <- erlang:system_info(alloc_util_allocators),
    {instance,N,Info} <- erlang:system_info({allocator,A}),
    {K,V} <- Info,
    K=/=options,
    W <- V,
    gettag(K,W)=/=drop].

aggregate() ->
  aggregate(indices()--[instance]).
aggregate(AggregateIndices) ->
  aggregate(AggregateIndices,[]).
aggregate(AggregateIndices,FilterTags) ->
  aggr(
    AggregateIndices,
    filter(
      FilterTags,
      getallocdata())).

calls() ->
  io:fwrite("~-20w~11w~11w~11w~11w~n",[allocator,alloc,dealloc,free,realloc]),
  lists:foreach(
    fun(L)->io:fwrite("~-20w~11w~11w~11w~11w~n",L)end,
    lists:foldl(
      fun({[X,alloc]  ,A},             T)  -> [[X,A,0,0,0]|T];
         ({[X,dealloc],D},[[X,A,0,0,0]|T]) -> [[X,A,D,0,0]|T];
         ({[X,free]   ,F},[[X,A,D,0,0]|T]) -> [[X,A,D,F,0]|T];
         ({[X,realloc],R},[[X,A,D,F,0]|T]) -> [[X,A,D,F,R]|T]
      end,
      [],
      lists:sort(atop:aggregate([allocator,info],[calls])))).

allocs() ->
  allocs(allocd).
allocs(SortCol) ->
  allocs(SortCol,[allocator,group]).
allocs(SortCol,AggregateIndices) ->
  printh(AggregateIndices,columns()),
  lists:foreach(
    fun print/1,
    lists:reverse(
      lists:keysort(
        column2index(SortCol),
        lists:foldl(
          fun present/2,
          [],
          aggregate(AggregateIndices++[type,info],[size]))))).

printh(AggInds,Cols) ->
  io:fwrite("~-33w~8w~8w~8w~n",[AggInds]++tl(Cols)).
print({K,T,U,F}) ->
  io:fwrite("~-33w~8.2f~8.2f~8.2f~n",[K,T,U,F]).

present({K,V},A) ->
  case {A,lists:reverse(K)} of
    {[{NK,V0}|A0],[size,blocks|NK]} -> [format(NK,V,V0)|A0];
    {[{NK,V0}|A0],[size,_|NK]}      -> [{NK,V+V0}|A0];
    {_,[size,_|NK]}                 -> [{NK,V}|A]
  end.

format(NK,V,V0) ->
  {lists:reverse(NK),V0/1024/1024,V/1024/1024,divide(V,V0)}.

divide(_V,0) -> 1.0;
divide(V,V0) -> V/V0.

filter([],PL) -> PL;
filter(Tags,PL) ->
  lists:filter(
    fun({K,_})->
        lists:member(true,[lists:member(T,K) || T <- Tags])
    end,
    PL).

aggr(IndexTags,List) ->
  Inds = [tag2index(T) || T <- IndexTags],
  lists:foldl(mkaggfun(Inds),[],lists:sort(mksortfun(Inds),List)).

columns() -> [what,allocd,used,frac].
indices() -> [allocator,instance,group,type,info].

column2index(X) -> x2index(X,columns()).
tag2index(X) -> x2index(X,indices()).

{'fraction_non_alphanumeric': 0.22983472241842068, 'fraction_numerical': 0.028252578047746855, 'mean_word_length': 3.0736478711162256, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5470341', 'n_tokens_mistral': 2951, 'n_tokens_neox': 2846, 'n_words': 369}