<commit_before>%% -------------------------------------------------------------------
%%
%% Copyright (c) 2007-2011 Basho Technologies, Inc.  All Rights Reserved.
%%
%% -------------------------------------------------------------------

%% @doc Module for choosing and manipulating lists (well, orddict) of manifests

-module(riak_moss_manifest).

-include("riak_moss.hrl").
-ifdef(TEST).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-endif.

%% export Public API
-export([new/2,
         active_manifest/1,
         mark_overwritten/1,
         pending_delete_manifests/1,
         prune/1,
         prune/2,
         upgrade_wrapped_manifests/1,
         upgrade_manifest/1]).

%%%===================================================================
%%% API
%%%===================================================================

%% @doc Return a new orddict of manifest (only
%% one in this case). Used when storing something
%% in Riak when the previous GET returned notfound,
%% so we're (maybe) creating a new object.
-spec new(binary(), lfs_manifest()) -> term().
new(UUID, Manifest) ->
    orddict:store(UUID, Manifest, orddict:new()).

%% @doc Return the current active manifest
%% from an orddict of manifests.
-spec active_manifest(term()) -> {ok, lfs_manifest()} | {error, no_active_manifest}.
active_manifest(Manifests) ->
    case lists:foldl(fun most_recent_active_manifest/2, no_active_manifest, orddict_values(Manifests)) of
        no_active_manifest ->
            {error, no_active_manifest};
        Manifest ->
            {ok, Manifest}
    end.

%% @doc Mark all active manifests
%% that are not "the most active"
%% as pending_delete
-spec mark_overwritten(term()) -> term().
mark_overwritten(Manifests) ->
    case active_manifest(Manifests) of
        {error, no_active_manifest} ->
            Manifests;
        {ok, Active} ->
            orddict:map(fun(_Key, Value) ->
                    if
                        Value == Active ->
                            Active;
                        Value?MANIFEST.state == active ->
                            Value?MANIFEST{state=pending_delete,
                                                 delete_marked_time=erlang:now()};
                        true ->
                            Value
                    end end,
                    Manifests)
    end.

%% @doc Return the current `pending_delete' manifests
%% from an orddict of manifests.
-spec pending_delete_manifests(term()) -> [lfs_manifest()].
pending_delete_manifests(Manifests) ->
    lists:filter(fun pending_delete_manifest/1, Manifests).

-spec prune(list(lfs_manifest())) -> list(lfs_manifest()).
prune(Manifests) ->
    prune(Manifests, erlang:now()).

-spec prune(list(lfs_manifest()), erlang:timestamp()) -> list(lfs_manifest()).
prune(Manifests, Time) ->
    [KV || {_K, V}=KV <- Manifests, not (needs_pruning(V, Time))].

upgrade_wrapped_manifests(ListofOrdDicts) ->
    DictMapFun = fun (_Key, Value) -> upgrade_manifest(Value) end,
    MapFun = fun (Value) -> orddict:map(DictMapFun, Value) end,
    lists:map(MapFun, ListofOrdDicts).

%% TODO:
%% not sure if there is a better spec for this,
%% since the input record could be one of several<commit_msg>Add funcs to changing manifest state to pending_del and scheduled_del<commit_after>%% -------------------------------------------------------------------
%%
%% Copyright (c) 2007-2011 Basho Technologies, Inc.  All Rights Reserved.
%%
%% -------------------------------------------------------------------

%% @doc Module for choosing and manipulating lists (well, orddict) of manifests

-module(riak_moss_manifest).

-include("riak_moss.hrl").
-ifdef(TEST).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-endif.

%% export Public API
-export([new/2,
         active_manifest/1,
         active_manifests/1,
         mark_overwritten/1,
         mark_pending_delete/2,
         mark_scheduled_delete/2,
         pending_delete_manifests/1,
         prune/1,
         prune/2,
         upgrade_wrapped_manifests/1,
         upgrade_manifest/1]).

%%%===================================================================
%%% API
%%%===================================================================

%% @doc Return a new orddict of manifest (only
%% one in this case). Used when storing something
%% in Riak when the previous GET returned notfound,
%% so we're (maybe) creating a new object.
-spec new(binary(), lfs_manifest()) -> term().
new(UUID, Manifest) ->
    orddict:store(UUID, Manifest, orddict:new()).

%% @doc Return the current active manifest
%% from an orddict of manifests.
-spec active_manifest(term()) -> {ok, lfs_manifest()} | {error, no_active_manifest}.
active_manifest(Manifests) ->
    case lists:foldl(fun most_recent_active_manifest/2, no_active_manifest, orddict_values(Manifests)) of
        no_active_manifest ->
            {error, no_active_manifest};
        Manifest ->
            {ok, Manifest}
    end.

%% @doc Return a list of all manifests in the
%% active state
-spec active_manifests(term()) -> [lfs_manifest()].
active_manifests(Manifests) ->
    IsActive = fun (?MANIFEST{state=State}) -> State == active end,
    lists:filter(IsActive, orddict_values(Manifests)).

%% @doc Mark all active manifests
%% that are not "the most active"
%% as pending_delete
-spec mark_overwritten(term()) -> term().
mark_overwritten(Manifests) ->
    case active_manifest(Manifests) of
        {error, no_active_manifest} ->
            Manifests;
        {ok, Active} ->
            orddict:map(fun(_Key, Value) ->
                    if
                        Value == Active ->
                            Active;
                        Value?MANIFEST.state == active ->
                            Value?MANIFEST{state=pending_delete,
                                                 delete_marked_time=erlang:now()};
                        true ->
                            Value
                    end end,
                    Manifests)
    end.

-spec mark_pending_delete(orddict:orddict(), list(binary())) ->
    orddict:orddict().
mark_pending_delete(Manifests, UUIDsToMark) ->
    MapFun = fun(K, V) ->
            case lists:member(K, UUIDsToMark) of
                true ->
                    V?MANIFEST{state=pending_delete};
                false ->
                    V
            end
    end,
    orddict:map(MapFun, Manifests).

-spec mark_scheduled_delete(orddict:orddict(), list(binary())) ->
    orddict:orddict().
mark_scheduled_delete(Manifests, UUIDsToMark) ->
    MapFun = fun(K, V) ->
            case lists:member(K, UUIDsToMark) of
                true ->
                    V?MANIFEST{state=scheduled_delete};
                false ->
                    V
            end
    end,
    orddict:map(MapFun, Manifests).

%% @doc Return the current `pending_delete' manifests
%% from an orddict of manifests.
-spec pending_delete_manifests(term()) -> [lfs_manifest()].
pending_delete_manifests(Manifests) ->
    lists:filter(fun pending_delete_manifest/1, Manifests).

-spec prune(list(lfs_manifest())) -> list(lfs_manifest()).
prune(Manifests) ->
    prune(Manifests, erlang:now()).

-spec prune(list(lfs_manifest()), erlang:timestamp()) -> list(lfs_manifest()).
prune(Manifests, Time) ->
    [KV || {_K, V}=KV <- Manifests, not (needs_pruning(V, Time))].

upgrade_wrapped_manifests(ListofOrdDicts) ->
    DictMapFun = fun (_Key, Value) -> upgrade_manifest(Value) end,
    MapFun = fun (Value) -> orddict:map(DictMapFun, Value) end,
    lists:map(MapFun, ListofOrdDicts).

%% TODO:
%% not sure if there is a better spec for this,
%% since the input record could be one of several
{'fraction_non_alphanumeric': 0.18636127017178553, 'fraction_numerical': 0.005075481520041645, 'mean_word_length': 2.3471254355400695, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5419109', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2198, 'n_words': 522}