<commit_before>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%% All rights reserved.

-module(stats_aggregator).
-include_lib("eunit/include/eunit.hrl").

-behaviour(gen_server).

-define(SAMPLE_SIZE, 61).
-define(STATS_MAXAGE, 750000). % max age of stats in the cache in microsecs
-define(TOPKEYS_MAXAGE, 5000000). % max age of topkeys in the cache in microsecs

%% API
-export([start_link/0,
         received_data/1,
         received_topkeys/4,
         get_stats/4,
         get_stats/3,
         get_stats/2,
         get_stats/1,
         get_topkeys/1,
         stats_age/3]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

-record(state, {vals, topkeys, cache, empty_ringdict}).

start_link() ->
    gen_server:start_link({global, ?MODULE}, ?MODULE, [], []).

init([]) ->
    timer:send_interval(5, garbage_collect),
    {ok, #state{vals=dict:new(), topkeys=dict:new(), cache=dict:new(),
                empty_ringdict=ringdict:new(?SAMPLE_SIZE)}}.

handle_call({get, Hostname, Port, Bucket, Count}, _From, State) ->
    Reply = (catch {ok, ringdict:to_dict(Count,
                             dict:fetch({Hostname, Port, Bucket},
                                        State#state.vals))}),
    {reply, Reply, State};
handle_call(Req = {get, Hostname, Port, Count}, _From, State) ->
    {Value, Cache} = cache_lookup(Req, State#state.cache, ?STATS_MAXAGE,
                                  fun () -> do_get(Hostname, Port, Count,
                                                   State) end),
    {reply, Value, State#state{cache=Cache}};
handle_call(Req = {get, Bucket, Count}, _From, State) ->
    {Value, Cache} = cache_lookup(Req, State#state.cache, ?STATS_MAXAGE,
                                  fun () -> do_get(Bucket, Count,
                                                   State) end),
    {reply, Value, State#state{cache=Cache}};
handle_call(Req = {get, Count}, _From, State) ->
    {Value, Cache} = cache_lookup(Req, State#state.cache, ?STATS_MAXAGE,
                                  fun () -> do_get(Count,
                                                   State) end),
    {reply, Value, State#state{cache=Cache}};<commit_msg>Bug 882: Do GC every 5 seconds rather than every 5 milliseconds<commit_after>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%% All rights reserved.

-module(stats_aggregator).
-include_lib("eunit/include/eunit.hrl").

-behaviour(gen_server).

-define(SAMPLE_SIZE, 61).
-define(STATS_MAXAGE, 750000). % max age of stats in the cache in microsecs
-define(TOPKEYS_MAXAGE, 5000000). % max age of topkeys in the cache in microsecs

%% API
-export([start_link/0,
         received_data/1,
         received_topkeys/4,
         get_stats/4,
         get_stats/3,
         get_stats/2,
         get_stats/1,
         get_topkeys/1,
         stats_age/3]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

-record(state, {vals, topkeys, cache, empty_ringdict}).

start_link() ->
    gen_server:start_link({global, ?MODULE}, ?MODULE, [], []).

init([]) ->
    timer:send_interval(5000, garbage_collect),
    {ok, #state{vals=dict:new(), topkeys=dict:new(), cache=dict:new(),
                empty_ringdict=ringdict:new(?SAMPLE_SIZE)}}.

handle_call({get, Hostname, Port, Bucket, Count}, _From, State) ->
    Reply = (catch {ok, ringdict:to_dict(Count,
                             dict:fetch({Hostname, Port, Bucket},
                                        State#state.vals))}),
    {reply, Reply, State};
handle_call(Req = {get, Hostname, Port, Count}, _From, State) ->
    {Value, Cache} = cache_lookup(Req, State#state.cache, ?STATS_MAXAGE,
                                  fun () -> do_get(Hostname, Port, Count,
                                                   State) end),
    {reply, Value, State#state{cache=Cache}};
handle_call(Req = {get, Bucket, Count}, _From, State) ->
    {Value, Cache} = cache_lookup(Req, State#state.cache, ?STATS_MAXAGE,
                                  fun () -> do_get(Bucket, Count,
                                                   State) end),
    {reply, Value, State#state{cache=Cache}};
handle_call(Req = {get, Count}, _From, State) ->
    {Value, Cache} = cache_lookup(Req, State#state.cache, ?STATS_MAXAGE,
                                  fun () -> do_get(Count,
                                                   State) end),
    {reply, Value, State#state{cache=Cache}};
{'fraction_non_alphanumeric': 0.14235731467308113, 'fraction_numerical': 0.017056636781106495, 'mean_word_length': 2.2347949080622347, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2326690', 'n_tokens_mistral': 1469, 'n_tokens_neox': 1406, 'n_words': 306}