<commit_before>%% ===================================================================
%% @author <NAME>
%% @copyright (C): 2013, ACK CYFRONET AGH
%% This software is released under the MIT license
%% cited in 'LICENSE.txt'.
%% @end
%% ===================================================================
%% @doc: FSLogic request handlers for regular files.
%% @end
%% ===================================================================
-module(fslogic_req_regular).
-author("<NAME>").

-include("oneprovider_modules/dao/dao.hrl").
-include("communication_protocol_pb.hrl").
-include("fuse_messages_pb.hrl").
-include("oneprovider_modules/fslogic/fslogic.hrl").
-include("oneprovider_modules/dao/dao_types.hrl").
-include("oneprovider_modules/fslogic/ranges_struct.hrl").
-include("oneprovider_modules/fslogic/fslogic_available_blocks.hrl").
-include_lib("ctool/include/logging.hrl").

%% API
-export([get_file_location/2, get_file_location/3, get_new_file_location/3,
    register_file_block/3, update_file_block_map/2, update_file_block_map/3,
    create_file_ack/1, file_not_used/1, renew_file_location/1]).

%% ====================================================================
%% API functions
%% ====================================================================


%% get_file_location/2
%% ====================================================================
%% @doc Gets file location (implicit file open operation).
%% @end
-spec get_file_location(File :: string() | file_doc(), OpenMode :: string()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_file_location(FileDoc = #db_document{record = #file{}}, ?UNSPECIFIED_MODE) ->
    get_file_location(FileDoc, none, ?UNSPECIFIED_MODE, false).


%% get_file_location/3
%% ====================================================================
%% @doc Gets file location (implicit file open operation). Allows to force-select ClusterProxy helper.
%% @end
-spec get_file_location(File :: string() | file_doc(), OpenMode :: string(), ForceClusterProxy :: boolean()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_file_location(FullFileName, OpenMode, ForceClusterProxy) when is_list(FullFileName) ->
    {ok, FileDoc} = fslogic_objects:get_file(FullFileName),
    get_file_location(FileDoc, FullFileName, OpenMode, ForceClusterProxy).

%% get_file_location/4
%% ====================================================================
%% @doc Gets file location (implicit file open operation).
%% @end
-spec get_file_location(FileDoc :: file_doc(), FullFileName :: string(), OpenMode :: string(), ForceClusterProxy :: boolean()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_file_location(FileDoc, FullFileName, OpenMode, ForceClusterProxy) ->
    ?info("get_file_location(~p, ~p, ~p, ~p)", [FileDoc, FullFileName, OpenMode, ForceClusterProxy]),
    Validity = ?LOCATION_VALIDITY,
    case FileDoc#db_document.record#file.type of
        ?REG_TYPE -> ok;
        UnSuppType ->
            ?error("Unsupported operation: get_file_location for file [UUID ~p] with type: ~p", [FileDoc#db_document.uuid, UnSuppType]),
            throw(?VENOTSUP)
    end,

    FileId = FileDoc#db_document.uuid,
    case dao_vfs:list_file_locations(FileId) of
        {ok, []} -> create_file_location_for_remote_file(FullFileName, FileId);
        _ -> ok
    end,

    fslogic_file:fix_storage_owner(FileDoc),

    {ok, UserDoc} = fslogic_objects:get_user(),
    ok = fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, list_to_existing_atom(OpenMode)),

    % cache all permissions
    case FileDoc#db_document.record#file.perms of
        Mask when (Mask band (?RWE_USR_PERM bor ?RWE_GRP_PERM bor ?RWE_OTH_PERM)) == 0 ->
            fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, rdwr),
            fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, execute),
            fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, delete);
        _ -> ok
    end,

    {ok, _} = fslogic_objects:save_file_descriptor(fslogic_context:get_protocol_version(), FileDoc#db_document.uuid, fslogic_context:get_fuse_id(), Validity),

    #db_document{record = FileLoc} = fslogic_file:get_file_local_location_doc(FileDoc),

    {ok, #space_info{space_id = SpaceId}} = fslogic_utils:get_space_info_for_path(FullFileName),

    {ok, #db_document{record = Storage}} = fslogic_objects:get_storage({uuid, FileLoc#file_location.storage_uuid}),

    {SH, File_id} = fslogic_utils:get_sh_and_id(fslogic_context:get_fuse_id(), Storage, FileLoc#file_location.storage_file_id, SpaceId, ForceClusterProxy),

    #filelocation{storage_id = Storage#storage_info.id, file_id = File_id, validity = Validity,
        storage_helper_name = SH#storage_helper_info.name, storage_helper_args = SH#storage_helper_info.init_args,
        available = get_blockavailability(FileDoc)}.


%% get_new_file_location/3
%% ====================================================================
%% @doc Gets new file location (implicit mknod operation).
%% @end
-spec get_new_file_location(FullFileName :: string(), Mode :: non_neg_integer(), ForceClusterProxy :: boolean()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_new_file_location(FullFileName, Mode, ForceClusterProxy) ->
    ?info("get_new_file_location(FullFileName ~p, Mode: ~p, ForceClusterProxy: ~p)", [FullFileName, Mode, ForceClusterProxy]),

    NewFileName = fslogic_path:basename(FullFileName),
    ParentFileName = fslogic_path:strip_path_leaf(FullFileName),
    {ok, #db_document{record = #file{}} = ParentDoc} = fslogic_objects:get_file(ParentFileName),

    {ok, UserDoc} = fslogic_objects:get_user(),
    FileBaseName = fslogic_path:get_user_file_name(FullFileName, UserDoc),

    ok = fslogic_perms:check_file_perms(FileBaseName, UserDoc, ParentDoc, write),

    {ok, #space_info{space_id = SpaceId} = SpaceInfo} = fslogic_utils:get_space_info_for_path(FullFileName),

    {ok, StorageList} = dao_lib:apply(dao_vfs, list_storage, [], fslogic_context:get_protocol_version()),
    #db_document{uuid = UUID, record = #storage_info{} = Storage} = fslogic_storage:select_storage(fslogic_context:get_fuse_id(), StorageList),
    SHI = fslogic_storage:get_sh_for_fuse(?CLUSTER_FUSE_ID, Storage),
    FileId = fslogic_storage:get_new_file_id(SpaceInfo, FileBaseName, UserDoc, SHI, fslogic_context:get_protocol_version()),

    {ok, UserID} = fslogic_context:get_user_id(),

    CTime = utils:time(),

    FileUUID = dao_helper:gen_uuid(),
    FileRecordInit = #file{type = ?REG_TYPE, name = NewFileName, uid = UserID, parent = ParentDoc#db_document.uuid, perms = Mode, created = false},
    %% Async *times update
    FileRecord = fslogic_meta:update_meta_attr(FileRecordInit, times, {0, 0, 0}),

    FileLocation = #file_location{file_id = FileUUID, storage_uuid = UUID, storage_file_id = FileId},
    {ok, LocationId} = dao_lib:apply(dao_vfs, save_file_location, [FileLocation], fslogic_context:get_protocol_version()),

    RemoteLocation = #available_blocks{file_id = FileUUID, provider_id = cluster_manager_lib:get_provider_id()},
    fslogic_available_blocks:call({save_available_blocks, RemoteLocation}),

    Validity = ?LOCATION_VALIDITY,
    FCreateStatus = dao_lib:apply(dao_vfs, save_new_file, [FullFileName, FileRecord, FileUUID], fslogic_context:get_protocol_version()),

    case FCreateStatus of
        {ok, {waiting_file, ExistingWFile}} ->
            ok = dao_lib:apply(dao_vfs, remove_file_location, [LocationId], fslogic_context:get_protocol_version()),
            ok = dao_lib:apply(dao_vfs, remove_available_blocks, [RemoteLocationId], fslogic_context:get_protocol_version()),

            ExistingWFileUUID = ExistingWFile#db_document.uuid,
            fslogic_meta:update_parent_ctime(FileBaseName, CTime),
            {ok, _} = fslogic_objects:save_file_descriptor(fslogic_context:get_protocol_version(), ExistingWFileUUID, fslogic_context:get_fuse_id(), Validity),

            #db_document{record = ExistingWFileLocation} =  fslogic_file:get_file_local_location_doc(ExistingWFileUUID),

            {ok, #db_document{record = ExistingWFileStorage}} = fslogic_objects:get_storage({uuid, ExistingWFileLocation#file_location.storage_uuid}),
            {SH, File_id2} = fslogic_utils:get_sh_and_id(fslogic_context:get_fuse_id(), ExistingWFileStorage, ExistingWFileLocation#file_location.storage_file_id, SpaceId, ForceClusterProxy),
            #storage_helper_info{name = ExistingWFileStorageSHName, init_args = ExistingWFileStorageSHArgs} = SH,
            #filelocation{storage_id = ExistingWFileStorage#storage_info.id, file_id = File_id2, validity = Validity, storage_helper_name = ExistingWFileStorageSHName, storage_helper_args = ExistingWFileStorageSHArgs, available = []};
        {ok, FileUUID} ->
            %% @todo: hack party! dbsync requires file to exist before syncing file_meta<commit_msg>bugfix<commit_after>%% ===================================================================
%% @author <NAME>ta
%% @copyright (C): 2013, ACK CYFRONET AGH
%% This software is released under the MIT license
%% cited in 'LICENSE.txt'.
%% @end
%% ===================================================================
%% @doc: FSLogic request handlers for regular files.
%% @end
%% ===================================================================
-module(fslogic_req_regular).
-author("<NAME>").

-include("oneprovider_modules/dao/dao.hrl").
-include("communication_protocol_pb.hrl").
-include("fuse_messages_pb.hrl").
-include("oneprovider_modules/fslogic/fslogic.hrl").
-include("oneprovider_modules/dao/dao_types.hrl").
-include("oneprovider_modules/fslogic/ranges_struct.hrl").
-include("oneprovider_modules/fslogic/fslogic_available_blocks.hrl").
-include_lib("ctool/include/logging.hrl").

%% API
-export([get_file_location/2, get_file_location/3, get_new_file_location/3,
    register_file_block/3, update_file_block_map/2, update_file_block_map/3,
    create_file_ack/1, file_not_used/1, renew_file_location/1]).

%% ====================================================================
%% API functions
%% ====================================================================


%% get_file_location/2
%% ====================================================================
%% @doc Gets file location (implicit file open operation).
%% @end
-spec get_file_location(File :: string() | file_doc(), OpenMode :: string()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_file_location(FileDoc = #db_document{record = #file{}}, ?UNSPECIFIED_MODE) ->
    get_file_location(FileDoc, none, ?UNSPECIFIED_MODE, false).


%% get_file_location/3
%% ====================================================================
%% @doc Gets file location (implicit file open operation). Allows to force-select ClusterProxy helper.
%% @end
-spec get_file_location(File :: string() | file_doc(), OpenMode :: string(), ForceClusterProxy :: boolean()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_file_location(FullFileName, OpenMode, ForceClusterProxy) when is_list(FullFileName) ->
    {ok, FileDoc} = fslogic_objects:get_file(FullFileName),
    get_file_location(FileDoc, FullFileName, OpenMode, ForceClusterProxy).

%% get_file_location/4
%% ====================================================================
%% @doc Gets file location (implicit file open operation).
%% @end
-spec get_file_location(FileDoc :: file_doc(), FullFileName :: string(), OpenMode :: string(), ForceClusterProxy :: boolean()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_file_location(FileDoc, FullFileName, OpenMode, ForceClusterProxy) ->
    ?info("get_file_location(~p, ~p, ~p, ~p)", [FileDoc, FullFileName, OpenMode, ForceClusterProxy]),
    Validity = ?LOCATION_VALIDITY,
    case FileDoc#db_document.record#file.type of
        ?REG_TYPE -> ok;
        UnSuppType ->
            ?error("Unsupported operation: get_file_location for file [UUID ~p] with type: ~p", [FileDoc#db_document.uuid, UnSuppType]),
            throw(?VENOTSUP)
    end,

    FileId = FileDoc#db_document.uuid,
    case dao_vfs:list_file_locations(FileId) of
        {ok, []} -> create_file_location_for_remote_file(FullFileName, FileId);
        _ -> ok
    end,

    fslogic_file:fix_storage_owner(FileDoc),

    {ok, UserDoc} = fslogic_objects:get_user(),
    ok = fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, list_to_existing_atom(OpenMode)),

    % cache all permissions
    case FileDoc#db_document.record#file.perms of
        Mask when (Mask band (?RWE_USR_PERM bor ?RWE_GRP_PERM bor ?RWE_OTH_PERM)) == 0 ->
            fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, rdwr),
            fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, execute),
            fslogic_perms:check_file_perms(FullFileName, UserDoc, FileDoc, delete);
        _ -> ok
    end,

    {ok, _} = fslogic_objects:save_file_descriptor(fslogic_context:get_protocol_version(), FileDoc#db_document.uuid, fslogic_context:get_fuse_id(), Validity),

    #db_document{record = FileLoc} = fslogic_file:get_file_local_location_doc(FileDoc),

    {ok, #space_info{space_id = SpaceId}} = fslogic_utils:get_space_info_for_path(FullFileName),

    {ok, #db_document{record = Storage}} = fslogic_objects:get_storage({uuid, FileLoc#file_location.storage_uuid}),

    {SH, File_id} = fslogic_utils:get_sh_and_id(fslogic_context:get_fuse_id(), Storage, FileLoc#file_location.storage_file_id, SpaceId, ForceClusterProxy),

    #filelocation{storage_id = Storage#storage_info.id, file_id = File_id, validity = Validity,
        storage_helper_name = SH#storage_helper_info.name, storage_helper_args = SH#storage_helper_info.init_args,
        available = get_blockavailability(FileDoc)}.


%% get_new_file_location/3
%% ====================================================================
%% @doc Gets new file location (implicit mknod operation).
%% @end
-spec get_new_file_location(FullFileName :: string(), Mode :: non_neg_integer(), ForceClusterProxy :: boolean()) ->
    #filelocation{} | no_return().
%% ====================================================================
get_new_file_location(FullFileName, Mode, ForceClusterProxy) ->
    ?info("get_new_file_location(FullFileName ~p, Mode: ~p, ForceClusterProxy: ~p)", [FullFileName, Mode, ForceClusterProxy]),

    NewFileName = fslogic_path:basename(FullFileName),
    ParentFileName = fslogic_path:strip_path_leaf(FullFileName),
    {ok, #db_document{record = #file{}} = ParentDoc} = fslogic_objects:get_file(ParentFileName),

    {ok, UserDoc} = fslogic_objects:get_user(),
    FileBaseName = fslogic_path:get_user_file_name(FullFileName, UserDoc),

    ok = fslogic_perms:check_file_perms(FileBaseName, UserDoc, ParentDoc, write),

    {ok, #space_info{space_id = SpaceId} = SpaceInfo} = fslogic_utils:get_space_info_for_path(FullFileName),

    {ok, StorageList} = dao_lib:apply(dao_vfs, list_storage, [], fslogic_context:get_protocol_version()),
    #db_document{uuid = UUID, record = #storage_info{} = Storage} = fslogic_storage:select_storage(fslogic_context:get_fuse_id(), StorageList),
    SHI = fslogic_storage:get_sh_for_fuse(?CLUSTER_FUSE_ID, Storage),
    FileId = fslogic_storage:get_new_file_id(SpaceInfo, FileBaseName, UserDoc, SHI, fslogic_context:get_protocol_version()),

    {ok, UserID} = fslogic_context:get_user_id(),

    CTime = utils:time(),

    FileUUID = dao_helper:gen_uuid(),
    FileRecordInit = #file{type = ?REG_TYPE, name = NewFileName, uid = UserID, parent = ParentDoc#db_document.uuid, perms = Mode, created = false},
    %% Async *times update
    FileRecord = fslogic_meta:update_meta_attr(FileRecordInit, times, {0, 0, 0}),

    FileLocation = #file_location{file_id = FileUUID, storage_uuid = UUID, storage_file_id = FileId},
    {ok, LocationId} = dao_lib:apply(dao_vfs, save_file_location, [FileLocation], fslogic_context:get_protocol_version()),

    RemoteLocation = #available_blocks{file_id = FileUUID, provider_id = cluster_manager_lib:get_provider_id()},
    {ok, RemoteLocationId} = fslogic_available_blocks:call({save_available_blocks, RemoteLocation}),

    Validity = ?LOCATION_VALIDITY,
    FCreateStatus = dao_lib:apply(dao_vfs, save_new_file, [FullFileName, FileRecord, FileUUID], fslogic_context:get_protocol_version()),

    case FCreateStatus of
        {ok, {waiting_file, ExistingWFile}} ->
            ok = dao_lib:apply(dao_vfs, remove_file_location, [LocationId], fslogic_context:get_protocol_version()),
            ok = dao_lib:apply(dao_vfs, remove_available_blocks, [RemoteLocationId], fslogic_context:get_protocol_version()),

            ExistingWFileUUID = ExistingWFile#db_document.uuid,
            fslogic_meta:update_parent_ctime(FileBaseName, CTime),
            {ok, _} = fslogic_objects:save_file_descriptor(fslogic_context:get_protocol_version(), ExistingWFileUUID, fslogic_context:get_fuse_id(), Validity),

            #db_document{record = ExistingWFileLocation} =  fslogic_file:get_file_local_location_doc(ExistingWFileUUID),

            {ok, #db_document{record = ExistingWFileStorage}} = fslogic_objects:get_storage({uuid, ExistingWFileLocation#file_location.storage_uuid}),
            {SH, File_id2} = fslogic_utils:get_sh_and_id(fslogic_context:get_fuse_id(), ExistingWFileStorage, ExistingWFileLocation#file_location.storage_file_id, SpaceId, ForceClusterProxy),
            #storage_helper_info{name = ExistingWFileStorageSHName, init_args = ExistingWFileStorageSHArgs} = SH,
            #filelocation{storage_id = ExistingWFileStorage#storage_info.id, file_id = File_id2, validity = Validity, storage_helper_name = ExistingWFileStorageSHName, storage_helper_args = ExistingWFileStorageSHArgs, available = []};
        {ok, FileUUID} ->
            %% @todo: hack party! dbsync requires file to exist before syncing file_meta
{'fraction_non_alphanumeric': 0.20739602765280368, 'fraction_numerical': 0.002523867003182267, 'mean_word_length': 6.255971337579618, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5103922', 'n_tokens_mistral': 5719, 'n_tokens_neox': 5450, 'n_words': 917}