<commit_before>-module(ice_cache).

%% ice_cache interface exports
-export([create/0, delete/0]).
-export([find/5, add/6]).

%%------------------------------------------------------------------------------
%% @doc Create a named ets table which represents the cache
%%------------------------------------------------------------------------------
create() ->
  ice_dtree:new().

%%------------------------------------------------------------------------------
%% @doc Delete the named ets table which represents the cache
%%------------------------------------------------------------------------------
delete() ->
  ice_dtree:delete().

%%------------------------------------------------------------------------------
%% @doc Find an identifier with a specific context K restricted by the domain D
%%------------------------------------------------------------------------------
find(X, K, D, {Id0, _} = W0, _T) ->
  KD = lists:keysort(1, ice_sets:restrict_domain(K, D)),
  case ice_dtree:lookup({X,KD}) of
    [] ->
%%      io:format(user, "Inserting X = ~p, KD = ~p, {calc, ~p}~n", [X, KD, W0]),
      true = ice_dtree:insert({X,KD,{calc,W0}}),
      {{calc,W0}, 0};
    {calc, {Id1,_} = W1} = V ->
      case lists:prefix(Id1, Id0) of
	true ->
	  hang;
	false ->
	  {V, 0}
      end;
    V ->
      {V, 0}
  end.

%%------------------------------------------------------------------------------
%% @doc Add an {identifier, context, value} to the cache
%%------------------------------------------------------------------------------
add(X, K, D, W, T, V1) ->
  KD = lists:keysort(1, ice_sets:restrict_domain(K, D)),
  case ice_dtree:lookup({X,KD}) of
    [] ->
      %% 
      find(X, K, D, W, T),
      add(X, K, D, W, T, V1);
    {calc, W} ->
      case V1 of
	V1 when is_list(V1) ->
	  true = ice_dtree:insert({X,KD,{i,V1,[]}});
	V1 ->
	  true = ice_dtree:insert({X,KD,V1})
      end,
      {V1, 0};
    {calc, _} = Thr ->
%%      io:format(user, "X = ~p, KD = ~p is a different thread...~n", [X, KD]),
      {V1, 0};
    ExistingV ->
%%      io:format(user, "Received other = ~p~n", [Other]),
      {ExistingV, 0}
  end.
<commit_msg>Add comment re issue in concurrent cache<commit_after>-module(ice_cache).

%% ice_cache interface exports
-export([create/0, delete/0]).
-export([find/5, add/6]).

%%------------------------------------------------------------------------------
%% @doc Create a named ets table which represents the cache
%%------------------------------------------------------------------------------
create() ->
  ice_dtree:new().

%%------------------------------------------------------------------------------
%% @doc Delete the named ets table which represents the cache
%%------------------------------------------------------------------------------
delete() ->
  ice_dtree:delete().

%%------------------------------------------------------------------------------
%% @doc Find an identifier with a specific context K restricted by the domain D
%%------------------------------------------------------------------------------
find(X, K, D, {Id0, _} = W0, _T) ->
  KD = lists:keysort(1, ice_sets:restrict_domain(K, D)),
  case ice_dtree:lookup({X,KD}) of
    [] ->
%%      io:format(user, "Inserting X = ~p, KD = ~p, {calc, ~p}~n", [X, KD, W0]),
      true = ice_dtree:insert({X,KD,{calc,W0}}),
      %% FIXME Lookup must atomically write calc if needed, otherwise
      %% concurrent find()s for the same {X,KD} would independently
      %% start computing the same value and write (different) calc,
      %% violating the assumption of add().
      {{calc,W0}, 0};
    {calc, {Id1,_} = W1} = V ->
      case lists:prefix(Id1, Id0) of
	true ->
	  hang;
	false ->
	  {V, 0}
      end;
    V ->
      {V, 0}
  end.

%%------------------------------------------------------------------------------
%% @doc Add an {identifier, context, value} to the cache
%%------------------------------------------------------------------------------
add(X, K, D, W, T, V1) ->
  KD = lists:keysort(1, ice_sets:restrict_domain(K, D)),
  case ice_dtree:lookup({X,KD}) of
    [] ->
      %% 
      find(X, K, D, W, T),
      add(X, K, D, W, T, V1);
    {calc, W} ->
      case V1 of
	V1 when is_list(V1) ->
	  true = ice_dtree:insert({X,KD,{i,V1,[]}});
	V1 ->
	  true = ice_dtree:insert({X,KD,V1})
      end,
      {V1, 0};
    {calc, _} = Thr ->
%%      io:format(user, "X = ~p, KD = ~p is a different thread...~n", [X, KD]),
      {V1, 0};
    ExistingV ->
%%      io:format(user, "Received other = ~p~n", [Other]),
      {ExistingV, 0}
  end.

{'fraction_non_alphanumeric': 0.4176354463698179, 'fraction_numerical': 0.01359947356876508, 'mean_word_length': 1.1438645980253879, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6672913', 'n_tokens_mistral': 1398, 'n_tokens_neox': 1333, 'n_words': 367}