<commit_before>%% -------------------------------------------------------------------
%%
%% mi: Merge-Index Data Store
%%
%% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved.
%%
%% -------------------------------------------------------------------
-module(mi_buffer).
-author("<NAME> <<EMAIL>>").
-include("merge_index.hrl").
-export([
    new/1,
    filename/1,
    close_filehandle/1,
    delete/1,
    filesize/1,
    size/1,
    write/7, write/2,
    info/4,
    iterator/1, iterator/4, iterators/6
]).

-ifdef(TEST).
-ifdef(EQC).
-include_lib("eqc/include/eqc.hrl").
-endif.
-include_lib("eunit/include/eunit.hrl").
-endif.


-record(buffer, {
    filename,
    handle,
    table,
    size
}).

%%% Creates a disk-based append-mode buffer file with support for a
%%% sorted iterator.

%% Open a new buffer. Returns a buffer structure.
new(Filename) ->
    %% Open the existing buffer file...
    filelib:ensure_dir(Filename),
    {ok, WriteOpts} = application:get_env(merge_index, buffer_write_options),
    {ok, FH} = file:open(Filename, [read, write, raw, binary] ++ WriteOpts),

    %% Read into an ets table...
    Table = ets:new(buffer, [duplicate_bag, public]),
    open_inner(FH, Table),
    {ok, Size} = file:position(FH, cur),

    %% Return the buffer.
    #buffer { filename=Filename, handle=FH, table=Table, size=Size }.

open_inner(FH, Table) ->
    case read_value(FH) of
        {ok, {Index, Field, Term, Value, Props, TS}} ->
            write_to_ets(Table, [{Index, Field, Term, Value, Props, TS}]),
            open_inner(FH, Table);
        eof ->
            ok
    end.

filename(Buffer) ->
    Buffer#buffer.filename.

delete(Buffer) ->
    ets:delete(Buffer#buffer.table),
    close_filehandle(Buffer),
    file:delete(Buffer#buffer.filename),
    file:delete(Buffer#buffer.filename ++ ".deleted"),
    ok.

close_filehandle(Buffer) ->
    file:close(Buffer#buffer.handle).

%% Return the current size of the buffer file.
filesize(Buffer) ->
    Buffer#buffer.size.

size(Buffer) ->
    ets:info(Buffer#buffer.table, size).

%% Write the value to the buffer.
%% Returns the new buffer structure.
write(Index, Field, Term, Value, Props, TS, Buffer) ->
    write([{Index, Field, Term, Value, Props, TS}], Buffer).

write(Postings, Buffer) ->
    %% Write to file...
    FH = Buffer#buffer.handle,
    BytesWritten = write_to_file(FH, Postings),

    %% Return a new buffer with a new tree and size...
    write_to_ets(Buffer#buffer.table, Postings),

    %% Return the new buffer.
    Buffer#buffer {
        size = (BytesWritten + Buffer#buffer.size)
    }.

%% Return the number of results under this IFT.
info(Index, Field, Term, Buffer) ->
    Spec = [{{{Index, Field, Term, '_'}, '_', '_'}, [], [true]}],
    ets:select_count(Buffer#buffer.table, Spec).

%% Return an iterator that traverses the entire buffer.
iterator(Buffer) ->
    Table = Buffer#buffer.table,
    List1 = ets:tab2list(Table),
    List2 = [{I,F,T,V,P,K} || {{I,F,T},V,P,K} <- List1],
    List3 = lists:sort(fun term_compare_fun/2, List2),
    fun() -> iterate_list(List3) end.

%% Return an iterator that traverses a range of the buffer.
iterator(Index, Field, Term, Buffer) ->
    Table = Buffer#buffer.table,
    List1 = ets:lookup(Table, {Index, Field, Term}),
    List2 = [{V,P,K} || {_Key,V,P,K} <- List1],
    List3 = lists:sort(fun value_compare_fun/2, List2),
    fun() -> iterate_list(List3) end.

%% Return a list of iterators over a range.
iterators(Index, Field, StartTerm, EndTerm, Size, Buffer) ->
    Table = Buffer#buffer.table,
    Keys = mi_utils:ets_keys(Table),
    Filter = fun(Key) ->
                     Key >= {Index, Field, StartTerm} 
                         andalso 
                         Key =< {Index, Field, EndTerm}
                         andalso
                         (Size == all orelse erlang:size(element(3, Key)) == Size)
        end,<commit_msg>Change approach to creating buffer iterator, reduce memory thrashing.<commit_after>%% -------------------------------------------------------------------
%%
%% mi: Merge-Index Data Store
%%
%% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved.
%%
%% -------------------------------------------------------------------
-module(mi_buffer).
-author("<NAME> <<EMAIL>>").
-include("merge_index.hrl").
-export([
    new/1,
    filename/1,
    close_filehandle/1,
    delete/1,
    filesize/1,
    size/1,
    write/7, write/2,
    info/4,
    iterator/1, iterator/4, iterators/6
]).

-ifdef(TEST).
-ifdef(EQC).
-include_lib("eqc/include/eqc.hrl").
-endif.
-include_lib("eunit/include/eunit.hrl").
-endif.


-record(buffer, {
    filename,
    handle,
    table,
    size
}).

%%% Creates a disk-based append-mode buffer file with support for a
%%% sorted iterator.

%% Open a new buffer. Returns a buffer structure.
new(Filename) ->
    %% Open the existing buffer file...
    filelib:ensure_dir(Filename),
    {ok, WriteOpts} = application:get_env(merge_index, buffer_write_options),
    {ok, FH} = file:open(Filename, [read, write, raw, binary] ++ WriteOpts),

    %% Read into an ets table...
    Table = ets:new(buffer, [duplicate_bag, public]),
    open_inner(FH, Table),
    {ok, Size} = file:position(FH, cur),

    %% Return the buffer.
    #buffer { filename=Filename, handle=FH, table=Table, size=Size }.

open_inner(FH, Table) ->
    case read_value(FH) of
        {ok, {Index, Field, Term, Value, Props, TS}} ->
            write_to_ets(Table, [{Index, Field, Term, Value, Props, TS}]),
            open_inner(FH, Table);
        eof ->
            ok
    end.

filename(Buffer) ->
    Buffer#buffer.filename.

delete(Buffer) ->
    ets:delete(Buffer#buffer.table),
    close_filehandle(Buffer),
    file:delete(Buffer#buffer.filename),
    file:delete(Buffer#buffer.filename ++ ".deleted"),
    ok.

close_filehandle(Buffer) ->
    file:close(Buffer#buffer.handle).

%% Return the current size of the buffer file.
filesize(Buffer) ->
    Buffer#buffer.size.

size(Buffer) ->
    ets:info(Buffer#buffer.table, size).

%% Write the value to the buffer.
%% Returns the new buffer structure.
write(Index, Field, Term, Value, Props, TS, Buffer) ->
    write([{Index, Field, Term, Value, Props, TS}], Buffer).

write(Postings, Buffer) ->
    %% Write to file...
    FH = Buffer#buffer.handle,
    BytesWritten = write_to_file(FH, Postings),

    %% Return a new buffer with a new tree and size...
    write_to_ets(Buffer#buffer.table, Postings),

    %% Return the new buffer.
    Buffer#buffer {
        size = (BytesWritten + Buffer#buffer.size)
    }.

%% Return the number of results under this IFT.
info(Index, Field, Term, Buffer) ->
    Spec = [{{{Index, Field, Term, '_'}, '_', '_'}, [], [true]}],
    ets:select_count(Buffer#buffer.table, Spec).

%% Return an iterator that traverses the entire buffer.
iterator(Buffer) ->
    Table = Buffer#buffer.table,
    Keys = lists:sort(mi_utils:ets_keys(Table)),
    fun() -> iterate_keys(Table, Keys) end.
    
iterate_keys(Table, [Key|Keys]) ->
    Results1 = ets:lookup(Table, Key),
    Results2 = [{I,F,T,V,P,K} || {{I,F,T},V,P,K} <- Results1],
    Results3 = lists:sort(fun term_compare_fun/2, Results2),
    iterate_keys_1(Table, Keys, Results3);
iterate_keys(_, []) ->
    eof.
iterate_keys_1(Table, Keys, [Result|Results]) ->
    {Result, fun() -> iterate_keys_1(Table, Keys, Results) end};
iterate_keys_1(Table, Keys, []) ->
    iterate_keys(Table, Keys).
    
%% Return an iterator that traverses a range of the buffer.
iterator(Index, Field, Term, Buffer) ->
    Table = Buffer#buffer.table,
    List1 = ets:lookup(Table, {Index, Field, Term}),
    List2 = [{V,P,K} || {_Key,V,P,K} <- List1],
    List3 = lists:sort(fun value_compare_fun/2, List2),
    fun() -> iterate_list(List3) end.

%% Return a list of iterators over a range.
iterators(Index, Field, StartTerm, EndTerm, Size, Buffer) ->
    Table = Buffer#buffer.table,
    Keys = mi_utils:ets_keys(Table),
    Filter = fun(Key) ->
                     Key >= {Index, Field, StartTerm} 
                         andalso 
                         Key =< {Index, Field, EndTerm}
                         andalso
                         (Size == all orelse erlang:size(element(3, Key)) == Size)
        end,
{'fraction_non_alphanumeric': 0.18406393799951562, 'fraction_numerical': 0.009082102203923468, 'mean_word_length': 2.6287346221441124, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5609286', 'n_tokens_mistral': 2758, 'n_tokens_neox': 2645, 'n_words': 669}