<commit_before>%% -------------------------------------------------------------------
%%
%% Copyright (c) 2007-2011 Basho Technologies, Inc.  All Rights Reserved.
%%
%% -------------------------------------------------------------------

%% @doc Module for choosing and manipulating lists (well, orddict) of manifests

-module(riak_moss_manifest).

-include("riak_moss.hrl").
-ifdef(TEST).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-endif.

%% export Public API
-export([new/2,
         active_manifest/1,
         active_manifests/1,
         overwritten_UUIDs/1,
         mark_pending_delete/2,
         mark_scheduled_delete/2,
         pending_delete_manifests/1,
         upgrade_wrapped_manifests/1,
         upgrade_manifest/1]).

%%%===================================================================
%%% API
%%%===================================================================

%% @doc Return a new orddict of manifest (only
%% one in this case). Used when storing something
%% in Riak when the previous GET returned notfound,
%% so we're (maybe) creating a new object.
-spec new(binary(), lfs_manifest()) -> term().
new(UUID, Manifest) ->
    orddict:store(UUID, Manifest, orddict:new()).

%% @doc Return the current active manifest
%% from an orddict of manifests.
-spec active_manifest(orddict:orddict()) -> {ok, lfs_manifest()} | {error, no_active_manifest}.
active_manifest(Manifests) ->
    case lists:foldl(fun most_recent_active_manifest/2, no_active_manifest, orddict_values(Manifests)) of
        no_active_manifest ->
            {error, no_active_manifest};
        Manifest ->
            {ok, Manifest}
    end.

%% @doc Return a list of all manifests in the
%% active state
-spec active_manifests(term()) -> [lfs_manifest()].
active_manifests(Manifests) ->
    IsActive =
        fun ({_, ?MANIFEST{state=State}}) ->
                State == active orelse State == writing
        end,
    lists:filter(IsActive, Manifests).

%% @doc Mark all active manifests
%% that are not "the most active"
%% as pending_delete
-spec overwritten_UUIDs(term()) -> term().
overwritten_UUIDs(Manifests) ->
    case active_manifest(Manifests) of
        {error, no_active_manifest} ->
            FoldFun =
                fun ({_, ?MANIFEST{state=State}}, Acc) when State =:= writing ->
                        Acc;
                    ({UUID, _}, Acc) ->
                        [UUID | Acc]
                end;
        {ok, Active} ->
            FoldFun = fun ({UUID, Elem}, Acc) ->
                              case Elem of
                                  Active ->
                                      Acc;
                                  Elem=?MANIFEST{state=active} ->
                                      [UUID | Acc];
                                  Elem=?MANIFEST{state=writing} ->
                                      [UUID | Acc];
                                  _ ->
                                      Acc
                              end
                      end
    end,
    lists:foldl(FoldFun, [], orddict:to_list(Manifests)).

%% @doc Return `Manifests' with the manifests in
%% `UUIDsToMark' with their state changed to
%% `pending_delete'
-spec mark_pending_delete(orddict:orddict(), list(binary())) ->
    orddict:orddict().
mark_pending_delete(Manifests, UUIDsToMark) ->
    MapFun = fun(K, V) ->
            case lists:member(K, UUIDsToMark) of
                true ->
                    V?MANIFEST{state=pending_delete,
                               delete_marked_time=os:timestamp()};
                false ->
                    V
            end
    end,
    orddict:map(MapFun, Manifests).

%% @doc Return `Manifests' with the manifests in
%% `UUIDsToMark' with their state changed to
%% `scheduled_delete'
-spec mark_scheduled_delete(orddict:orddict(), list(binary())) ->
    orddict:orddict().
mark_scheduled_delete(Manifests, UUIDsToMark) ->
    MapFun = fun(K, V) ->
            case lists:member(K, UUIDsToMark) of
                true ->
                    V?MANIFEST{state=scheduled_delete,
                               scheduled_delete_time=os:timestamp()};
                false ->
                    V
            end
    end,
    orddict:map(MapFun, Manifests).

%% @doc Return the current `pending_delete' manifests
%% from an orddict of manifests.
-spec pending_delete_manifests(term()) -> [lfs_manifest()].
pending_delete_manifests(Manifests) ->
    lists:filter(fun pending_delete_manifest/1, Manifests).

upgrade_wrapped_manifests(ListofOrdDicts) ->
    DictMapFun = fun (_Key, Value) -> upgrade_manifest(Value) end,
    MapFun = fun (Value) -> orddict:map(DictMapFun, Value) end,
    lists:map(MapFun, ListofOrdDicts).

%% TODO:
%% not sure if there is a better spec for this,
%% since the input record could be one of several
%% versions.

%% @doc Upgrade the manifest to the most recent
%% version of the manifest record. This is so that
%% _most_ of the codebase only has to deal with
%% the most recent version of the record.
-spec upgrade_manifest(term()) -> lfs_manifest().
upgrade_manifest(#lfs_manifest_v2{block_size=BlockSize,
                                 bkey=Bkey,
                                 metadata=Metadata,
                                 created=Created,
                                 uuid=UUID,
                                 content_length=ContentLength,
                                 content_type=ContentType,
                                 content_md5=ContentMd5,
                                 state=State,
                                 write_start_time=WriteStartTime,
                                 last_block_written_time=LastBlockWrittenTime,
                                 write_blocks_remaining=WriteBlocksRemaining,
                                 delete_marked_time=DeleteMarkedTime,
                                 last_block_deleted_time=LastBlockDeletedTime,
                                 delete_blocks_remaining=DeleteBlocksRemaining,
                                 acl=Acl,
                                 props=Properties,
                                 cluster_id=ClusterID}) ->

    ?MANIFEST{block_size=BlockSize,
              bkey=Bkey,
              metadata=Metadata,
              created=Created,
              uuid=UUID,
              content_length=ContentLength,
              content_type=ContentType,
              content_md5=ContentMd5,
              state=State,
              write_start_time=WriteStartTime,
              last_block_written_time=LastBlockWrittenTime,
              write_blocks_remaining=WriteBlocksRemaining,
              delete_marked_time=DeleteMarkedTime,
              last_block_deleted_time=LastBlockDeletedTime,
              delete_blocks_remaining=DeleteBlocksRemaining,
              acl=Acl,
              props=Properties,
              cluster_id=ClusterID};

upgrade_manifest(?MANIFEST{}=M) ->
    M.

%%%===================================================================
%%% Internal functions
%%%===================================================================

orddict_values(OrdDict) ->
    %% orddict's are by definition
    %% represented as lists, so no
    %% need to call orddict:to_list,
    %% which actually is the identity
    %% func
    [V || {_K, V} <- OrdDict].

most_recent_active_manifest(Manifest=?MANIFEST{state=active}, no_active_manifest) -><commit_msg>Change active_manifests function in riak_moss_manifest to active_and_writing_manifests.<commit_after>%% -------------------------------------------------------------------
%%
%% Copyright (c) 2007-2011 Basho Technologies, Inc.  All Rights Reserved.
%%
%% -------------------------------------------------------------------

%% @doc Module for choosing and manipulating lists (well, orddict) of manifests

-module(riak_moss_manifest).

-include("riak_moss.hrl").
-ifdef(TEST).
-compile(export_all).
-include_lib("eunit/include/eunit.hrl").
-endif.

%% export Public API
-export([new/2,
         active_manifest/1,
         active_and_writing_manifests/1,
         overwritten_UUIDs/1,
         mark_pending_delete/2,
         mark_scheduled_delete/2,
         pending_delete_manifests/1,
         upgrade_wrapped_manifests/1,
         upgrade_manifest/1]).

%%%===================================================================
%%% API
%%%===================================================================

%% @doc Return a new orddict of manifest (only
%% one in this case). Used when storing something
%% in Riak when the previous GET returned notfound,
%% so we're (maybe) creating a new object.
-spec new(binary(), lfs_manifest()) -> term().
new(UUID, Manifest) ->
    orddict:store(UUID, Manifest, orddict:new()).

%% @doc Return the current active manifest
%% from an orddict of manifests.
-spec active_manifest(orddict:orddict()) -> {ok, lfs_manifest()} | {error, no_active_manifest}.
active_manifest(Manifests) ->
    case lists:foldl(fun most_recent_active_manifest/2, no_active_manifest, orddict_values(Manifests)) of
        no_active_manifest ->
            {error, no_active_manifest};
        Manifest ->
            {ok, Manifest}
    end.

%% @doc Return a list of all manifests in the
%% `active' or `writing' state
-spec active_and_writing_manifests(term()) -> [lfs_manifest()].
active_and_writing_manifests(Manifests) ->
    filter_manifests_by_state(Manifests, [active, writing]).

%% @doc Mark all active manifests
%% that are not "the most active"
%% as pending_delete
-spec overwritten_UUIDs(term()) -> term().
overwritten_UUIDs(Manifests) ->
    case active_manifest(Manifests) of
        {error, no_active_manifest} ->
            FoldFun =
                fun ({_, ?MANIFEST{state=State}}, Acc) when State =:= writing ->
                        Acc;
                    ({UUID, _}, Acc) ->
                        [UUID | Acc]
                end;
        {ok, Active} ->
            FoldFun = fun ({UUID, Elem}, Acc) ->
                              case Elem of
                                  Active ->
                                      Acc;
                                  Elem=?MANIFEST{state=active} ->
                                      [UUID | Acc];
                                  Elem=?MANIFEST{state=writing} ->
                                      [UUID | Acc];
                                  _ ->
                                      Acc
                              end
                      end
    end,
    lists:foldl(FoldFun, [], orddict:to_list(Manifests)).

%% @doc Return `Manifests' with the manifests in
%% `UUIDsToMark' with their state changed to
%% `pending_delete'
-spec mark_pending_delete(orddict:orddict(), list(binary())) ->
    orddict:orddict().
mark_pending_delete(Manifests, UUIDsToMark) ->
    MapFun = fun(K, V) ->
            case lists:member(K, UUIDsToMark) of
                true ->
                    V?MANIFEST{state=pending_delete,
                               delete_marked_time=os:timestamp()};
                false ->
                    V
            end
    end,
    orddict:map(MapFun, Manifests).

%% @doc Return `Manifests' with the manifests in
%% `UUIDsToMark' with their state changed to
%% `scheduled_delete'
-spec mark_scheduled_delete(orddict:orddict(), list(binary())) ->
    orddict:orddict().
mark_scheduled_delete(Manifests, UUIDsToMark) ->
    MapFun = fun(K, V) ->
            case lists:member(K, UUIDsToMark) of
                true ->
                    V?MANIFEST{state=scheduled_delete,
                               scheduled_delete_time=os:timestamp()};
                false ->
                    V
            end
    end,
    orddict:map(MapFun, Manifests).

%% @doc Return the current `pending_delete' manifests
%% from an orddict of manifests.
-spec pending_delete_manifests(term()) -> [lfs_manifest()].
pending_delete_manifests(Manifests) ->
    lists:filter(fun pending_delete_manifest/1, Manifests).

upgrade_wrapped_manifests(ListofOrdDicts) ->
    DictMapFun = fun (_Key, Value) -> upgrade_manifest(Value) end,
    MapFun = fun (Value) -> orddict:map(DictMapFun, Value) end,
    lists:map(MapFun, ListofOrdDicts).

%% TODO:
%% not sure if there is a better spec for this,
%% since the input record could be one of several
%% versions.

%% @doc Upgrade the manifest to the most recent
%% version of the manifest record. This is so that
%% _most_ of the codebase only has to deal with
%% the most recent version of the record.
-spec upgrade_manifest(term()) -> lfs_manifest().
upgrade_manifest(#lfs_manifest_v2{block_size=BlockSize,
                                 bkey=Bkey,
                                 metadata=Metadata,
                                 created=Created,
                                 uuid=UUID,
                                 content_length=ContentLength,
                                 content_type=ContentType,
                                 content_md5=ContentMd5,
                                 state=State,
                                 write_start_time=WriteStartTime,
                                 last_block_written_time=LastBlockWrittenTime,
                                 write_blocks_remaining=WriteBlocksRemaining,
                                 delete_marked_time=DeleteMarkedTime,
                                 last_block_deleted_time=LastBlockDeletedTime,
                                 delete_blocks_remaining=DeleteBlocksRemaining,
                                 acl=Acl,
                                 props=Properties,
                                 cluster_id=ClusterID}) ->

    ?MANIFEST{block_size=BlockSize,
              bkey=Bkey,
              metadata=Metadata,
              created=Created,
              uuid=UUID,
              content_length=ContentLength,
              content_type=ContentType,
              content_md5=ContentMd5,
              state=State,
              write_start_time=WriteStartTime,
              last_block_written_time=LastBlockWrittenTime,
              write_blocks_remaining=WriteBlocksRemaining,
              delete_marked_time=DeleteMarkedTime,
              last_block_deleted_time=LastBlockDeletedTime,
              delete_blocks_remaining=DeleteBlocksRemaining,
              acl=Acl,
              props=Properties,
              cluster_id=ClusterID};

upgrade_manifest(?MANIFEST{}=M) ->
    M.

%%%===================================================================
%%% Internal functions
%%%===================================================================

%% @doc Filter a list of manifests and accept only manifests whose
%% current state is specified in the `AcceptedStates' list.
-spec filter_manifests_by_state([lfs_manifest()], [atom()]) -> [lfs_manifest()].
filter_manifests_by_state(Manifests, AcceptedStates) ->
    AcceptManifest =
        fun ({_, ?MANIFEST{state=State}}) ->
                lists:member(State, AcceptedStates)
        end,
    lists:filter(AcceptManifest, Manifests).

orddict_values(OrdDict) ->
    %% orddict's are by definition
    %% represented as lists, so no
    %% need to call orddict:to_list,
    %% which actually is the identity
    %% func
    [V || {_K, V} <- OrdDict].

most_recent_active_manifest(Manifest=?MANIFEST{state=active}, no_active_manifest) ->
{'fraction_non_alphanumeric': 0.1462711975381392, 'fraction_numerical': 0.003142801021410332, 'mean_word_length': 1.8993925588458618, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6929662', 'n_tokens_mistral': 4229, 'n_tokens_neox': 4022, 'n_words': 881}