<commit_before>-module(windex).

-export([lookup/2, add/2, delete/2, deleteByValue/2, export/1]).
-compile(export_all).
%% ------------------------------------------------------------------
%% API Function Definitions
%% ------------------------------------------------------------------

lookup(Index, Key) ->
        KeyParts = binary:split(Key, <<"/">>, [global]),
        {ok, RootNode} = windex_db:fetch(primary_tree, Index),
        doLookup(KeyParts, RootNode).

add(Index, Rev) when is_map(Rev) ->
        RevTree = makeTree(Rev),
        NewTree = case windex_db:getIfExists(primary_tree, Index) of
                false -> RevTree;
                {ok, FullTree} -> mergeTrees(FullTree, RevTree)
        end,
        windex_db:store(primary_tree, Index, NewTree).

delete(Index, Rev) when is_map(Rev) ->
        RevTree = makeTree(Rev),
        {ok, FullTree} = windex_db:fetch(primary_tree, Index),
        windex_db:store(primary_tree, Index, demergeTrees(FullTree, RevTree)).

deleteByValue(Index, Value) -> ok.

export({SubTree, Data}) when is_map(SubTree)->
        #{
                data    => Data,
                subtree => maps:map(fun(_Key, Value) -> export(Value) end, SubTree)
        };
export(Index) ->
        {ok, RootNode} = windex_db:fetch(primary_tree, Index),
        export(RootNode).
%% ------------------------------------------------------------------
%% Internal Function Definitions
%% ------------------------------------------------------------------

doLookup([<<>> | Rest], Node) -> doLookup(Rest, Node);
doLookup([], Node) -> Node;
doLookup([Label | Rest], {Node, _Data}) when is_map(Node) ->
        NewNode = maps:get(Label, Node),
        doLookup(Rest, NewNode).

convertNode(Value) when is_map(Value) -> {doMakeTree(Value), []};
convertNode(Value) when is_list(Value) -> {#{}, Value}.

makeTree(Revision) when is_map(Revision) -> {doMakeTree(Revision), []}.
doMakeTree(Revision) when is_map(Revision) ->
        compactNode(maps:map(fun(_Key, Value) -> convertNode(Value) end, Revision)).

mergeTrees({LsubTree, Ldata}, {RsubTree, Rdata}) ->
        NewData = lists:umerge(Ldata, Rdata),<commit_msg>lets make sure that data nodes are uniq sort, just for safety<commit_after>-module(windex).

-export([lookup/2, add/2, delete/2, deleteByValue/2, export/1]).
-compile(export_all).
%% ------------------------------------------------------------------
%% API Function Definitions
%% ------------------------------------------------------------------

lookup(Index, Key) ->
        KeyParts = binary:split(Key, <<"/">>, [global]),
        {ok, RootNode} = windex_db:fetch(primary_tree, Index),
        doLookup(KeyParts, RootNode).

add(Index, Rev) when is_map(Rev) ->
        RevTree = makeTree(Rev),
        NewTree = case windex_db:getIfExists(primary_tree, Index) of
                false -> RevTree;
                {ok, FullTree} -> mergeTrees(FullTree, RevTree)
        end,
        windex_db:store(primary_tree, Index, NewTree).

delete(Index, Rev) when is_map(Rev) ->
        RevTree = makeTree(Rev),
        {ok, FullTree} = windex_db:fetch(primary_tree, Index),
        windex_db:store(primary_tree, Index, demergeTrees(FullTree, RevTree)).

deleteByValue(Index, Value) -> ok.

export({SubTree, Data}) when is_map(SubTree)->
        #{
                data    => Data,
                subtree => maps:map(fun(_Key, Value) -> export(Value) end, SubTree)
        };
export(Index) ->
        {ok, RootNode} = windex_db:fetch(primary_tree, Index),
        export(RootNode).
%% ------------------------------------------------------------------
%% Internal Function Definitions
%% ------------------------------------------------------------------

doLookup([<<>> | Rest], Node) -> doLookup(Rest, Node);
doLookup([], Node) -> Node;
doLookup([Label | Rest], {Node, _Data}) when is_map(Node) ->
        NewNode = maps:get(Label, Node),
        doLookup(Rest, NewNode).

convertNode(Value) when is_map(Value) -> {doMakeTree(Value), []};
convertNode(Value) when is_list(Value) -> {#{}, lists:usort(Value)}.

makeTree(Revision) when is_map(Revision) -> {doMakeTree(Revision), []}.
doMakeTree(Revision) when is_map(Revision) ->
        compactNode(maps:map(fun(_Key, Value) -> convertNode(Value) end, Revision)).

mergeTrees({LsubTree, Ldata}, {RsubTree, Rdata}) ->
        NewData = lists:umerge(Ldata, Rdata),
{'fraction_non_alphanumeric': 0.26498277841561424, 'fraction_numerical': 0.002296211251435132, 'mean_word_length': 2.1069900142653353, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1821909', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1271, 'n_words': 262}