<commit_before>%%%-------------------------------------------------------------------
%%% File:      dmerkle.erl
%%% @author    <NAME> <> []
%%% @copyright 2008 <NAME>
%%% @doc  
%%%
%%% @end  
%%%
%%% @since 2008-08-03 by <NAME>
%%%-------------------------------------------------------------------
-module(dmerkle).
-author('<EMAIL>').

-behavior(gen_server).

-record(dmerkle, {tree,root}).

-include("common.hrl").
-include("dmerkle.hrl").

%% API
-export([open/1, open/2, equals/2, get_tree/1, count/2, count_trace/2, update/3, updatea/3, delete/2, deletea/2, leaves/1, find/2, visualized_find/2, key_diff/2, close/1, scan_for_empty/1, swap_tree/2]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

-ifdef(TEST).
-include("etest/dmerkle_test.erl").
-endif.

%%====================================================================
%% Public API
%%====================================================================

open(FileName) ->
  open(FileName, undefined).

open(FileName, BlockSize) -> 
  gen_server:start_link(?MODULE, {FileName,BlockSize}, []).

count_trace(Pid, Key) ->
  gen_server:call(Pid, {count_trace, Key}).

count(Pid, Level) ->
  gen_server:call(Pid, {count, Level}).

leaves(Pid) ->
  gen_server:call(Pid, leaves).

update(Key, Value, Pid) ->
  gen_server:call(Pid, {update, Key, Value}, infinity).
  
updatea(Key, Value, Pid) ->
  gen_server:cast(Pid, {update, Key, Value}).

equals(A, B) ->
  gen_server:call(A, hash) == gen_server:call(B, hash).

find(Key, Tree) ->
  gen_server:call(Tree, {find, Key}).
  
visualized_find(Key, Tree) ->
  gen_server:call(Tree, {visualized_find, Key}).

delete(Key, Tree) ->
  gen_server:call(Tree, {delete, Key}).
  
deletea(Key, Tree) ->
  gen_server:cast(Tree, {delete, Key}).

key_diff(TreeA, TreeB) ->
  gen_server:call(TreeA, {key_diff, TreeB}).

close(Tree) ->
  gen_server:cast(Tree, close).

scan_for_empty(Tree) ->
  gen_server:call(Tree, scan_for_empty).

get_tree(Tree) ->
  gen_server:call(Tree, get_tree).
  
get_state(Tree) ->
  gen_server:call(Tree, get_state).

swap_tree(OldTree, NewTree) ->
  NewFilename = gen_server:call(NewTree, filename),
  BlockSize = gen_server:call(OldTree, blocksize),
  OldFilename = gen_server:call(OldTree, filename),
  close(OldTree),
  close(NewTree),
  file:copy(block_server:index_name(NewFilename), block_server:index_name(OldFilename)),
  file:copy(block_server:key_name(NewFilename), block_server:key_name(OldFilename)),
  file:delete(block_server:index_name(NewFilename)),
  file:delete(block_server:key_name(NewFilename)),
  open(OldFilename, BlockSize).

  %%====================================================================
%% gen_server callbacks
%%====================================================================

%%--------------------------------------------------------------------<commit_msg>more aggressive GC in dmerkle and set the key diff timeout to infinity<commit_after>%%%-------------------------------------------------------------------
%%% File:      dmerkle.erl
%%% @author    <NAME> <> []
%%% @copyright 2008 <NAME>
%%% @doc  
%%%
%%% @end  
%%%
%%% @since 2008-08-03 by <NAME>
%%%-------------------------------------------------------------------
-module(dmerkle).
-author('cliff<EMAIL>').

-behavior(gen_server).

-record(dmerkle, {tree,root}).

-include("common.hrl").
-include("dmerkle.hrl").

%% API
-export([open/1, open/2, equals/2, get_tree/1, count/2, count_trace/2, update/3, updatea/3, delete/2, deletea/2, leaves/1, find/2, visualized_find/2, key_diff/2, close/1, scan_for_empty/1, swap_tree/2]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

-ifdef(TEST).
-include("etest/dmerkle_test.erl").
-endif.

%%====================================================================
%% Public API
%%====================================================================

open(FileName) ->
  open(FileName, undefined).

open(FileName, BlockSize) -> 
  gen_server:start_link(?MODULE, {FileName,BlockSize}, [{spawn_opt, [fullsweep_after, 10]}]).

count_trace(Pid, Key) ->
  gen_server:call(Pid, {count_trace, Key}).

count(Pid, Level) ->
  gen_server:call(Pid, {count, Level}).

leaves(Pid) ->
  gen_server:call(Pid, leaves).

update(Key, Value, Pid) ->
  gen_server:call(Pid, {update, Key, Value}, infinity).
  
updatea(Key, Value, Pid) ->
  gen_server:cast(Pid, {update, Key, Value}).

equals(A, B) ->
  gen_server:call(A, hash) == gen_server:call(B, hash).

find(Key, Tree) ->
  gen_server:call(Tree, {find, Key}).
  
visualized_find(Key, Tree) ->
  gen_server:call(Tree, {visualized_find, Key}).

delete(Key, Tree) ->
  gen_server:call(Tree, {delete, Key}).
  
deletea(Key, Tree) ->
  gen_server:cast(Tree, {delete, Key}).

key_diff(TreeA, TreeB) ->
  gen_server:call(TreeA, {key_diff, TreeB}, infinity).

close(Tree) ->
  gen_server:cast(Tree, close).

scan_for_empty(Tree) ->
  gen_server:call(Tree, scan_for_empty).

get_tree(Tree) ->
  gen_server:call(Tree, get_tree).
  
get_state(Tree) ->
  gen_server:call(Tree, get_state).

swap_tree(OldTree, NewTree) ->
  NewFilename = gen_server:call(NewTree, filename),
  BlockSize = gen_server:call(OldTree, blocksize),
  OldFilename = gen_server:call(OldTree, filename),
  close(OldTree),
  close(NewTree),
  file:copy(block_server:index_name(NewFilename), block_server:index_name(OldFilename)),
  file:copy(block_server:key_name(NewFilename), block_server:key_name(OldFilename)),
  file:delete(block_server:index_name(NewFilename)),
  file:delete(block_server:key_name(NewFilename)),
  open(OldFilename, BlockSize).

  %%====================================================================
%% gen_server callbacks
%%====================================================================

%%--------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.3155668358714044, 'fraction_numerical': 0.012182741116751269, 'mean_word_length': 3.877062706270627, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6464921', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1962, 'n_words': 322}