<commit_before>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2019 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% Module used for creation and validation of helper
%%% argument and user/group ctx maps.
%%% @end
%%%-------------------------------------------------------------------
-module(helper_params).
-author("<NAME>").


-include("modules/storage_file_manager/helpers/helpers.hrl").
-include("modules/datastore/datastore_models.hrl").
-include_lib("ctool/include/logging.hrl").

%% API
-export([validate_args/2, validate_user_ctx/2, validate_group_ctx/2]).
-export([default_admin_ctx/1]).

%% Onepanel RPC API
-export([prepare_helper_args/2, prepare_user_ctx_params/2]).

-type name() :: helper:name().
-type args() :: helper:args().
-type user_ctx() :: helper:user_ctx().
-type group_ctx() :: helper:group_ctx().
-type ctx() :: user_ctx() | group_ctx().

-type field() :: binary().
-type optional_field() :: {optional, field()}.
-type field_spec() :: field() | optional_field().


%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc
%% Translates storage params as specified by the user
%% (i.e. via Onepanel API when creating the storage)
%% into correct helper args.
%% The result will not contain any unknown fields
%% but is not verified to contain all required fields.
%% @end
%%--------------------------------------------------------------------
-spec prepare_helper_args(name(), args()) -> args().
prepare_helper_args(?S3_HELPER_NAME = HelperName, Params) ->
    Args = derive_scheme_from_url(Params),
    filter_fields(expected_helper_args(HelperName), Args);

prepare_helper_args(HelperName, Params) ->
    filter_fields(expected_helper_args(HelperName), Params).


%% @private
-spec derive_scheme_from_url(args()) -> args().
derive_scheme_from_url(#{<<"hostname">> := URL} = Params) ->
    {ok, UrlScheme, Host} = parse_url(URL),
    Scheme = case UrlScheme of
        https -> <<"https">>;
        _ -> <<"http">>
    end,
    Params#{<<"scheme">> => Scheme, <<"hostname">> => Host};
derive_scheme_from_url(Params) -> Params.


%%--------------------------------------------------------------------
%% @doc
%% Translates storage params as specified by the user
%% (i.e. via Onepanel API when creating the storage)
%% into correct user ctx.
%% The result will not contain any unknown fields
%% but is not verified to contain all required fields.
%% @end
%%--------------------------------------------------------------------
-spec prepare_user_ctx_params(name(), ctx()) -> ctx().
prepare_user_ctx_params(?WEBDAV_HELPER_NAME = HelperName, Params) ->
    Ctx1 = clear_unused_webdav_credentials(Params),
    Ctx2 = resolve_user_by_token(Ctx1),
    filter_fields(expected_user_ctx_params(HelperName), Ctx2);

prepare_user_ctx_params(HelperName, Params) ->
    filter_fields(expected_user_ctx_params(HelperName), Params).


%% @private
-spec clear_unused_webdav_credentials(user_ctx()) -> user_ctx().
clear_unused_webdav_credentials(#{<<"credentialsType">> := <<"none">>} = Params) ->
    maps:remove(<<"credentials">>, Params);
clear_unused_webdav_credentials(Params) -> Params.


%% @private
-spec resolve_user_by_token(user_ctx()) -> user_ctx().
resolve_user_by_token(#{<<"onedataAccessToken">> := Token} = Params) when
    byte_size(Token) > 0 ->
    {ok, UserId} = user_identity:get_or_fetch_user_id(Token),
    Params#{<<"adminId">> => UserId};
resolve_user_by_token(Params) -> Params.


%%--------------------------------------------------------------------
%% @doc
%% Translates storage params as specified by the user
%% (i.e. via Onepanel API when creating the storage)
%% into correct user ctx.
%% The result will not contain any unknown fields
%% but is not verified to contain all required fields.
%% @end
%%--------------------------------------------------------------------
-spec validate_args(name(), args()) ->
    ok | {error, Reason :: term()}.
validate_args(HelperName, Args) ->
    Fields = expected_helper_args(HelperName),
    validate_fields(Fields, Args).


%%--------------------------------------------------------------------
%% @doc
%% Checks whether user context is valid for the storage helper.
%% @end
%%--------------------------------------------------------------------
-spec validate_user_ctx(name(), user_ctx()) ->
    ok | {error, Reason :: term()}.
validate_user_ctx(StorageType = ?WEBDAV_HELPER_NAME, UserCtx) ->
    FieldsBase = expected_user_ctx_params(StorageType),
    Fields = case UserCtx of
        #{<<"credentialsType">> := <<"none">>} ->
            FieldsBase;
        #{<<"credentialsType">> := _} ->
            % make "credentials" required rather than optional
            [<<"credentials">> | remove_field(<<"credentials">>, FieldsBase)];
        _ ->
            FieldsBase
    end,
    validate_fields(Fields, UserCtx);

validate_user_ctx(StorageType, UserCtx) ->
    Fields = expected_user_ctx_params(StorageType),
    validate_fields(Fields, UserCtx).



%%--------------------------------------------------------------------
%% @doc
%% Checks whether user context is valid for the storage helper.
%% @end
%%--------------------------------------------------------------------
-spec validate_group_ctx(storage:helper(), group_ctx()) ->
    ok | {error, Reason :: term()}.
validate_group_ctx(#helper{name = ?POSIX_HELPER_NAME}, GroupCtx) ->
    validate_fields([<<"gid">>], GroupCtx);
validate_group_ctx(#helper{name = ?GLUSTERFS_HELPER_NAME}, GroupCtx) ->
    validate_fields([<<"gid">>], GroupCtx);
validate_group_ctx(#helper{name = ?NULL_DEVICE_HELPER_NAME}, GroupCtx) ->
    validate_fields([<<"gid">>], GroupCtx);
validate_group_ctx(#helper{name = ?WEBDAV_HELPER_NAME}, _GroupCtx) ->
    ok;
validate_group_ctx(#helper{name = HelperName}, _GroupCtx) ->
    {error, {group_ctx_not_supported, HelperName}}.


-spec default_admin_ctx(name()) -> user_ctx().
default_admin_ctx(HelperName) when
    HelperName == ?POSIX_HELPER_NAME;
    HelperName == ?NULL_DEVICE_HELPER_NAME;
    HelperName == ?GLUSTERFS_HELPER_NAME ->
    #{<<"uid">> => <<"0">>, <<"gid">> => <<"0">>};

default_admin_ctx(_) ->
    #{}.

%%%===================================================================
%%% Requirements
%%%===================================================================

%% @private
-spec expected_helper_args(name()) ->
    [field() | optional_field()].
expected_helper_args(?CEPH_HELPER_NAME) -> [
    <<"monitorHostname">>, <<"clusterName">>, <<"poolName">>,
    {optional, <<"timeout">>}];
expected_helper_args(?CEPHRADOS_HELPER_NAME) -> [
    <<"monitorHostname">>, <<"clusterName">>, <<"poolName">>,
    {optional, <<"timeout">>}, {optional, <<"blockSize">>}];
expected_helper_args(?POSIX_HELPER_NAME) -> [
    <<"mountPoint">>,
    {optional, <<"timeout">>}];
expected_helper_args(?S3_HELPER_NAME) -> [
    <<"hostname">>, <<"bucketName">>, <<"scheme">>,
    {optional, <<"timeout">>}, {optional, <<"signatureVersion">>},
    {optional, <<"blockSize">>}];
expected_helper_args(?SWIFT_HELPER_NAME) -> [
    <<"authUrl">>, <<"containerName">>, <<"tenantName">>,
    {optional, <<"timeout">>}, {optional, <<"blockSize">>}];
expected_helper_args(?GLUSTERFS_HELPER_NAME) -> [
    <<"volume">>, <<"hostname">>,
    {optional, <<"port">>}, {optional, <<"mountPoint">>},
    {optional, <<"transport">>}, {optional, <<"xlatorOptions">>},
    {optional, <<"timeout">>}, {optional, <<"blockSize">>}];
expected_helper_args(?WEBDAV_HELPER_NAME) -> [
    <<"endpoint">>,
    {optional, <<"oauth2IdP">>},
    {optional, <<"timeout">>}, {optional, <<"verifyServerCertificate">>},
    {optional, <<"authorizationHeader">>}, {optional, <<"rangeWriteSupport">>},
    {optional, <<"connectionPoolSize">>}, {optional, <<"maximumUploadSize">>},
    {optional, <<"fileMode">>}, {optional, <<"dirMode">>}];
expected_helper_args(?NULL_DEVICE_HELPER_NAME) -> [
    {optional, <<"timeout">>}, {optional, <<"latencyMin">>},
    {optional, <<"latencyMax">>}, {optional, <<"timeoutProbability">>},
    {optional, <<"filter">>},
    {optional, <<"simulatedFilesystemParameters">>},
    {optional, <<"simulatedFilesystemGrowSpeed">>}].


%%--------------------------------------------------------------------
%% @private
%% @doc<commit_msg>VFS-5597 Added s3 sync helper params<commit_after>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2019 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% Module used for creation and validation of helper
%%% argument and user/group ctx maps.
%%% @end
%%%-------------------------------------------------------------------
-module(helper_params).
-author("<NAME>").


-include("modules/storage_file_manager/helpers/helpers.hrl").
-include("modules/datastore/datastore_models.hrl").
-include_lib("ctool/include/logging.hrl").

%% API
-export([validate_args/2, validate_user_ctx/2, validate_group_ctx/2]).
-export([default_admin_ctx/1]).

%% Onepanel RPC API
-export([prepare_helper_args/2, prepare_user_ctx_params/2]).

-type name() :: helper:name().
-type args() :: helper:args().
-type user_ctx() :: helper:user_ctx().
-type group_ctx() :: helper:group_ctx().
-type ctx() :: user_ctx() | group_ctx().

-type field() :: binary().
-type optional_field() :: {optional, field()}.
-type field_spec() :: field() | optional_field().


%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc
%% Translates storage params as specified by the user
%% (i.e. via Onepanel API when creating the storage)
%% into correct helper args.
%% The result will not contain any unknown fields
%% but is not verified to contain all required fields.
%% @end
%%--------------------------------------------------------------------
-spec prepare_helper_args(name(), args()) -> args().
prepare_helper_args(?S3_HELPER_NAME = HelperName, Params) ->
    Args = derive_scheme_from_url(Params),
    filter_fields(expected_helper_args(HelperName), Args);

prepare_helper_args(HelperName, Params) ->
    filter_fields(expected_helper_args(HelperName), Params).


%% @private
-spec derive_scheme_from_url(args()) -> args().
derive_scheme_from_url(#{<<"hostname">> := URL} = Params) ->
    {ok, UrlScheme, Host} = parse_url(URL),
    Scheme = case UrlScheme of
        https -> <<"https">>;
        _ -> <<"http">>
    end,
    Params#{<<"scheme">> => Scheme, <<"hostname">> => Host};
derive_scheme_from_url(Params) -> Params.


%%--------------------------------------------------------------------
%% @doc
%% Translates storage params as specified by the user
%% (i.e. via Onepanel API when creating the storage)
%% into correct user ctx.
%% The result will not contain any unknown fields
%% but is not verified to contain all required fields.
%% @end
%%--------------------------------------------------------------------
-spec prepare_user_ctx_params(name(), ctx()) -> ctx().
prepare_user_ctx_params(?WEBDAV_HELPER_NAME = HelperName, Params) ->
    Ctx1 = clear_unused_webdav_credentials(Params),
    Ctx2 = resolve_user_by_token(Ctx1),
    filter_fields(expected_user_ctx_params(HelperName), Ctx2);

prepare_user_ctx_params(HelperName, Params) ->
    filter_fields(expected_user_ctx_params(HelperName), Params).


%% @private
-spec clear_unused_webdav_credentials(user_ctx()) -> user_ctx().
clear_unused_webdav_credentials(#{<<"credentialsType">> := <<"none">>} = Params) ->
    maps:remove(<<"credentials">>, Params);
clear_unused_webdav_credentials(Params) -> Params.


%% @private
-spec resolve_user_by_token(user_ctx()) -> user_ctx().
resolve_user_by_token(#{<<"onedataAccessToken">> := Token} = Params) when
    byte_size(Token) > 0 ->
    {ok, UserId} = user_identity:get_or_fetch_user_id(Token),
    Params#{<<"adminId">> => UserId};
resolve_user_by_token(Params) -> Params.


%%--------------------------------------------------------------------
%% @doc
%% Translates storage params as specified by the user
%% (i.e. via Onepanel API when creating the storage)
%% into correct user ctx.
%% The result will not contain any unknown fields
%% but is not verified to contain all required fields.
%% @end
%%--------------------------------------------------------------------
-spec validate_args(name(), args()) ->
    ok | {error, Reason :: term()}.
validate_args(HelperName, Args) ->
    Fields = expected_helper_args(HelperName),
    validate_fields(Fields, Args).


%%--------------------------------------------------------------------
%% @doc
%% Checks whether user context is valid for the storage helper.
%% @end
%%--------------------------------------------------------------------
-spec validate_user_ctx(name(), user_ctx()) ->
    ok | {error, Reason :: term()}.
validate_user_ctx(StorageType = ?WEBDAV_HELPER_NAME, UserCtx) ->
    FieldsBase = expected_user_ctx_params(StorageType),
    Fields = case UserCtx of
        #{<<"credentialsType">> := <<"none">>} ->
            FieldsBase;
        #{<<"credentialsType">> := _} ->
            % make "credentials" required rather than optional
            [<<"credentials">> | remove_field(<<"credentials">>, FieldsBase)];
        _ ->
            FieldsBase
    end,
    validate_fields(Fields, UserCtx);

validate_user_ctx(StorageType, UserCtx) ->
    Fields = expected_user_ctx_params(StorageType),
    validate_fields(Fields, UserCtx).



%%--------------------------------------------------------------------
%% @doc
%% Checks whether user context is valid for the storage helper.
%% @end
%%--------------------------------------------------------------------
-spec validate_group_ctx(storage:helper(), group_ctx()) ->
    ok | {error, Reason :: term()}.
validate_group_ctx(#helper{name = ?POSIX_HELPER_NAME}, GroupCtx) ->
    validate_fields([<<"gid">>], GroupCtx);
validate_group_ctx(#helper{name = ?GLUSTERFS_HELPER_NAME}, GroupCtx) ->
    validate_fields([<<"gid">>], GroupCtx);
validate_group_ctx(#helper{name = ?NULL_DEVICE_HELPER_NAME}, GroupCtx) ->
    validate_fields([<<"gid">>], GroupCtx);
validate_group_ctx(#helper{name = ?WEBDAV_HELPER_NAME}, _GroupCtx) ->
    ok;
validate_group_ctx(#helper{name = HelperName}, _GroupCtx) ->
    {error, {group_ctx_not_supported, HelperName}}.


-spec default_admin_ctx(name()) -> user_ctx().
default_admin_ctx(HelperName) when
    HelperName == ?POSIX_HELPER_NAME;
    HelperName == ?NULL_DEVICE_HELPER_NAME;
    HelperName == ?GLUSTERFS_HELPER_NAME ->
    #{<<"uid">> => <<"0">>, <<"gid">> => <<"0">>};

default_admin_ctx(_) ->
    #{}.

%%%===================================================================
%%% Requirements
%%%===================================================================

%% @private
-spec expected_helper_args(name()) ->
    [field() | optional_field()].
expected_helper_args(?CEPH_HELPER_NAME) -> [
    <<"monitorHostname">>, <<"clusterName">>, <<"poolName">>,
    {optional, <<"timeout">>}];
expected_helper_args(?CEPHRADOS_HELPER_NAME) -> [
    <<"monitorHostname">>, <<"clusterName">>, <<"poolName">>,
    {optional, <<"timeout">>}, {optional, <<"blockSize">>}];
expected_helper_args(?POSIX_HELPER_NAME) -> [
    <<"mountPoint">>,
    {optional, <<"timeout">>}];
expected_helper_args(?S3_HELPER_NAME) -> [
    <<"hostname">>, <<"bucketName">>, <<"scheme">>,
    {optional, <<"timeout">>}, {optional, <<"signatureVersion">>},
    {optional, <<"maximumCanonicalObjectSize">>},
    {optional, <<"fileMode">>}, {optional, <<"dirMode">>},
    {optional, <<"blockSize">>}];
expected_helper_args(?SWIFT_HELPER_NAME) -> [
    <<"authUrl">>, <<"containerName">>, <<"tenantName">>,
    {optional, <<"timeout">>}, {optional, <<"blockSize">>}];
expected_helper_args(?GLUSTERFS_HELPER_NAME) -> [
    <<"volume">>, <<"hostname">>,
    {optional, <<"port">>}, {optional, <<"mountPoint">>},
    {optional, <<"transport">>}, {optional, <<"xlatorOptions">>},
    {optional, <<"timeout">>}, {optional, <<"blockSize">>}];
expected_helper_args(?WEBDAV_HELPER_NAME) -> [
    <<"endpoint">>,
    {optional, <<"oauth2IdP">>},
    {optional, <<"timeout">>}, {optional, <<"verifyServerCertificate">>},
    {optional, <<"authorizationHeader">>}, {optional, <<"rangeWriteSupport">>},
    {optional, <<"connectionPoolSize">>}, {optional, <<"maximumUploadSize">>},
    {optional, <<"fileMode">>}, {optional, <<"dirMode">>}];
expected_helper_args(?NULL_DEVICE_HELPER_NAME) -> [
    {optional, <<"timeout">>}, {optional, <<"latencyMin">>},
    {optional, <<"latencyMax">>}, {optional, <<"timeoutProbability">>},
    {optional, <<"filter">>},
    {optional, <<"simulatedFilesystemParameters">>},
    {optional, <<"simulatedFilesystemGrowSpeed">>}].


%%--------------------------------------------------------------------
%% @private
%% @doc
{'fraction_non_alphanumeric': 0.307402687673772, 'fraction_numerical': 0.0026065801668211305, 'mean_word_length': 3.029171528588098, 'pattern_counts': {'":': 0, '<': 289, '<?xml version=': 0, '>': 387, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3017125', 'n_tokens_mistral': 5214, 'n_tokens_neox': 5091, 'n_words': 1008}