<commit_before>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%% All rights reserved.

-module(ns_log).

-include("ns_log.hrl").

% Number of recent log entries.
-define(RB_SIZE, 50).
-define(DUP_TIME, 300000000). % 300 secs in microsecs
-define(GC_TIME, 60000). % 60 secs in millisecs

-behaviour(gen_server).
-behavior(ns_log_categorizing).

%% API
-export([start_link/0]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

-export([log/3, log/4, recent/0, recent/1, recent_by_category/0, clear/0]).

-export([categorize/2, code_string/2]).

-export([ns_log_cat/1, ns_log_code_string/1]).

-include_lib("eunit/include/eunit.hrl").

-record(state, {recent, dedup}).

start_link() ->
    Result = gen_server:start_link({global, ?MODULE}, ?MODULE, [], []),
    ns_log:log(?MODULE, 4, "Log server started on node ~p~n", [node()]),
    Result.

init([]) ->
    timer:send_interval(?GC_TIME, garbage_collect),
    {ok, #state{recent=emptyRecent(), dedup=dict:new()}}.

emptyRecent() ->
    ringbuffer:new(?RB_SIZE).

% Request for recent items.
handle_call(recent, _From, State) ->
    Reply = ringbuffer:to_list(State#state.recent),
    {reply, Reply, State}.

% Inbound logging request.
handle_cast({log, Module, Code, Fmt, Args}, State = #state{dedup=Dedup}) ->
    Now = erlang:now(),
    Key = {Module, Code, Fmt, Args},
    case dict:find(Key, Dedup) of
    {ok, {Count, FirstSeen, LastSeen}} ->
        error_logger:info_msg("ns_log: suppressing duplicate log ~p:~p(~p) because it's been "
                           "seen ~p times in the past ~p secs (last seen ~p secs ago~n",
                           [Module, Code, lists:flatten(io_lib:format(Fmt, Args)),
                            Count+1, timer:now_diff(Now, FirstSeen) / 1000000,
                            timer:now_diff(Now, LastSeen) / 1000000]),
        Dedup2 = dict:store(Key, {Count+1, FirstSeen, Now}, Dedup),
        {noreply, State#state{dedup=Dedup2}};
    error ->
        State2 = do_log(Module, Code, Fmt, Args, State),
        Dedup2 = dict:store(Key, {0, Now, Now}, Dedup),
        {noreply, State2#state{dedup=Dedup2}}
    end;
handle_cast(clear, _State) ->
    error_logger:info_msg("Clearing log.~n", []),
    {noreply,  #state{recent=emptyRecent(), dedup=dict:new()}}.
% Not handling any other state.

% Nothing special.
handle_info(garbage_collect, State) ->
    {noreply, gc(State)};
handle_info(_Info, State) ->
    {noreply, State}.

terminate(_Reason, _State) ->
    ok.

code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%%--------------------------------------------------------------------
%%% Internal functions
%%--------------------------------------------------------------------

do_log(Module, Code, Fmt, Args, State) ->
    Category = categorize(Module, Code),
    NR = ringbuffer:add(#log_entry{module=Module, code=Code, msg=Fmt, args=Args,
                                   cat=Category,
                                   tstamp=erlang:now()},
                        State#state.recent),
    try gen_event:notify(ns_log_events, {ns_log, Category, Module, Code,
                         Fmt, Args})
    catch _:_Reason ->
        error_logger:info_msg("ns_log: message: ~p, ~p, ~p, ~p~n",
                              [Module, Code, Fmt, Args])
    end,
    State#state{recent=NR}.

gc(State = #state{dedup=Dupes}) ->
    {DupesList, State2} = gc(erlang:now(), dict:to_list(Dupes),
                                           [], State),
    State2#state{dedup=dict:from_list(DupesList)}.

gc(_Now, [], DupesList, State) -> {DupesList, State};
gc(Now, [{Key, Value} | Rest], DupesList, State) ->
     {Count, FirstSeen, _LastSeen} = Value,
     case timer:now_diff(Now, FirstSeen) >= ?DUP_TIME of
     true ->
         {Module, Code, Fmt, Args} = Key,
         State2 = case Count of<commit_msg>Revert "bug 856 - reapplying previous ns_log fix"<commit_after>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%% All rights reserved.

-module(ns_log).

-include("ns_log.hrl").

% Number of recent log entries.
-define(RB_SIZE, 50).
-define(DUP_TIME, 300000000). % 300 secs in microsecs
-define(GC_TIME, 60000). % 60 secs in millisecs

-behaviour(gen_server).
-behavior(ns_log_categorizing).

%% API
-export([start_link/0]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

-export([log/3, log/4, recent/0, recent/1, recent_by_category/0, clear/0]).

-export([categorize/2, code_string/2]).

-export([ns_log_cat/1, ns_log_code_string/1]).

-include_lib("eunit/include/eunit.hrl").

-record(state, {recent, dedup}).

start_link() ->
    Result = gen_server:start_link({global, ?MODULE}, ?MODULE, [], []),
    ns_log:log(?MODULE, 4, "Log server started on node ~p~n", [node()]),
    Result.

init([]) ->
    timer:send_interval(?GC_TIME, garbage_collect),
    {ok, #state{recent=emptyRecent(), dedup=dict:new()}}.

emptyRecent() ->
    ringbuffer:new(?RB_SIZE).

% Request for recent items.
handle_call(recent, _From, State) ->
    Reply = ringbuffer:to_list(State#state.recent),
    {reply, Reply, State}.

% Inbound logging request.
handle_cast({log, Module, Code, Fmt, Args}, State = #state{dedup=Dedup}) ->
    Now = erlang:now(),
    Key = {Module, Code, Fmt, Args},
    case dict:find(Key, Dedup) of
    {ok, {Count, FirstSeen, LastSeen}} ->
        error_logger:info_msg("ns_log: suppressing duplicate log ~p:~p(~p) because it's been "
                           "seen ~p times in the past ~p secs (last seen ~p secs ago~n",
                           [Module, Code, lists:flatten(io_lib:format(Fmt, Args)),
                            Count+1, timer:now_diff(Now, FirstSeen) / 1000000,
                            timer:now_diff(Now, LastSeen) / 1000000]),
        Dedup2 = dict:store(Key, {Count+1, FirstSeen, Now}, Dedup),
        {noreply, State#state{dedup=Dedup2}};
    error ->
        State2 = do_log(Module, Code, Fmt, Args, State),
        Dedup2 = dict:store(Key, {0, Now, Now}, Dedup),
        {noreply, State2#state{dedup=Dedup2}}
    end;
handle_cast(clear, _State) ->
    error_logger:info_msg("Clearing log.~n", []),
    {noreply,  #state{recent=emptyRecent(), dedup=dict:new()}}.
% Not handling any other state.

% Nothing special.
handle_info(garbage_collect, State) ->
    {noreply, gc(State)};
handle_info(_Info, State) ->
    {noreply, State}.

terminate(_Reason, _State) ->
    ok.

code_change(_OldVsn, State, _Extra) ->
    {ok, State}.

%%--------------------------------------------------------------------
%%% Internal functions
%%--------------------------------------------------------------------

do_log(Module, Code, Fmt, Args, State) ->
    Category = categorize(Module, Code),
    NR = ringbuffer:add(#log_entry{module=Module, code=Code, msg=Fmt, args=Args,
                                   cat=Category,
                                   tstamp=erlang:now()},
                        State#state.recent),
    try gen_event:notify(ns_log_events, {ns_log, Category, Module, Code,
                         Fmt, Args})
    catch _:Reason ->
        error_logger:error_msg("ns_log: unable to notify listeners "
                               "because of ~p~n", [Reason])
    end,
    State#state{recent=NR}.

gc(State = #state{dedup=Dupes}) ->
    {DupesList, State2} = gc(erlang:now(), dict:to_list(Dupes),
                                           [], State),
    State2#state{dedup=dict:from_list(DupesList)}.

gc(_Now, [], DupesList, State) -> {DupesList, State};
gc(Now, [{Key, Value} | Rest], DupesList, State) ->
     {Count, FirstSeen, _LastSeen} = Value,
     case timer:now_diff(Now, FirstSeen) >= ?DUP_TIME of
     true ->
         {Module, Code, Fmt, Args} = Key,
         State2 = case Count of
{'fraction_non_alphanumeric': 0.18947502548419978, 'fraction_numerical': 0.017966360856269112, 'mean_word_length': 2.554800724637681, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295539', 'n_tokens_mistral': 2799, 'n_tokens_neox': 2653, 'n_words': 568}