<commit_before>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C): 2016 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% This module processes file upload requests originating from GUI.
%%% Exposes an API required by lib ResumableJS that is used on client side.
%%% @end
%%%-------------------------------------------------------------------
-module(upload_handler).
-author("<NAME>").
-behaviour(cowboy_http_handler).

-include("global_definitions.hrl").
-include_lib("ctool/include/logging.hrl").

% Key of in-memory mapping of uploads kept in session.
-define(UPLOAD_MAP, upload_map).

% How many tries of creating a unique filename before failure.
-define(MAX_UNIQUE_FILENAME_COUNTER, 20).

% Maximum time a process will be waiting for chunk 1 to generate file handle.
-define(MAX_WAIT_FOR_FILE_HANDLE, 30000).

% Interval between retries to resolve file handle.
-define(INTERVAL_WAIT_FOR_FILE_HANDLE, 100).

%% Cowboy API
-export([init/3, handle/2, terminate/3]).
%% API
-export([upload_map_insert/2, upload_map_delete/1]).
-export([upload_map_lookup/1, upload_map_lookup/2]).
-export([clean_upload_map/0]).

%% ====================================================================
%% API functions
%% ====================================================================


%%--------------------------------------------------------------------
%% @doc
%% Inserts a UploadId - FileId pair into upload map.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_insert(UploadId :: term(), FileId :: term()) -> ok.
upload_map_insert(UploadId, FileId) ->
    Map = g_session:get_value(?UPLOAD_MAP, #{}),
    NewMap = maps:put(UploadId, FileId, Map),
    g_session:put_value(?UPLOAD_MAP, NewMap).


%%--------------------------------------------------------------------
%% @doc
%% Retrieves a FileId from upload map by UploadId.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_lookup(UploadId :: term()) -> FileId :: term().
upload_map_lookup(UploadId) ->
    upload_map_lookup(UploadId, undefined).


%%--------------------------------------------------------------------
%% @doc
%% Retrieves a FileId from upload map by UploadId or returns a default value.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_lookup(UploadId :: term(), Default :: term()) ->
    FileId :: term().
upload_map_lookup(UploadId, Default) ->
    Map = g_session:get_value(?UPLOAD_MAP, #{}),
    maps:get(UploadId, Map, Default).


%%--------------------------------------------------------------------
%% @doc
%% Deletes a UploadId - FileId pair from upload map.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_delete(UploadId :: term()) -> ok.
upload_map_delete(UploadId) ->
    Map = g_session:get_value(?UPLOAD_MAP, #{}),
    NewMap = maps:remove(UploadId, Map),
    g_session:put_value(?UPLOAD_MAP, NewMap).


%%--------------------------------------------------------------------
%% @doc
%% Cleans all upload-related mappings in session memory.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec clean_upload_map() -> ok.
clean_upload_map() ->
    g_session:put_value(?UPLOAD_MAP, #{}).


%% ====================================================================
%% Cowboy API functions
%% ====================================================================

%%--------------------------------------------------------------------
%% @doc
%% Cowboy handler callback.
%% @end
%%--------------------------------------------------------------------
-spec init({TransportName :: atom(), ProtocolName :: http},
    Req :: cowboy_req:req(), Opts :: any()) ->
    {ok, cowboy_req:req(), []}.
init(_Type, Req, _Opts) ->
    {ok, Req, []}.


%%--------------------------------------------------------------------
%% @doc
%% Handles an upload request.
%% @end
%%--------------------------------------------------------------------
-spec handle(term(), term()) -> {ok, cowboy_req:req(), term()}.
handle(Req, State) ->
    NewReq = handle_http_upload(Req),
    {ok, NewReq, State}.


%%--------------------------------------------------------------------
%% @doc
%% Cowboy handler callback, no cleanup needed
%% @end
%%--------------------------------------------------------------------
-spec terminate(term(), term(), term()) -> ok.
terminate(_Reason, _Req, _State) ->
    ok.


%% ====================================================================
%% Internal functions
%% ====================================================================

%%--------------------------------------------------------------------
%% @private
%% @doc
%% Asserts the validity of multipart POST request and proceeds with
%% parsing or returns an error. Returns list of parsed filed values and
%% file body.
%% @end
%%--------------------------------------------------------------------
-spec handle_http_upload(Req :: cowboy_req:req()) -> cowboy_req:req().
handle_http_upload(Req) ->
    % Try to retrieve user's session
    InitSession =
        try
            g_ctx:init(Req, false)
        catch _:_ ->
            % Error logging is done inside g_ctx:init
            error
        end,
    case InitSession of
        error ->
            g_ctx:reply(500, [], <<"">>);
        ok ->
            try
                NewReq = multipart(Req, []),
                g_ctx:set_cowboy_req(NewReq),
                g_ctx:reply(200, [], <<"">>)
            catch
                throw:{missing_param, _} ->
                    g_ctx:reply(500, [], <<"">>);
                throw:stream_file_error ->
                    g_ctx:reply(500, [], <<"">>);
                Type:Message ->
                    ?error_stacktrace("Error while processing file upload "
                    "from user ~p - ~p:~p",
                        [g_session:get_user_id(), Type, Message]),
                    % @todo VFS-1815 for now return 500,
                    % because retries are not stable
%%                    % Return 204 - resumable will retry the upload
%%                    g_ctx:reply(204, [], <<"">>)
                    g_ctx:reply(500, [], <<"">>)
            end
    end,
    g_ctx:finish().


%%--------------------------------------------------------------------
%% @private
%% @doc
%% Parses a multipart HTTP form.
%% @end
%%--------------------------------------------------------------------
-spec multipart(Req :: cowboy_req:req(), Params :: proplists:proplist()) ->
    cowboy_req:req().
multipart(Req, Params) ->
    {ok, UploadWriteSize} = application:get_env(?APP_NAME, upload_write_size),
    {ok, UploadReadTimeout} = application:get_env(?APP_NAME, upload_read_timeout),
    {ok, UploadReadSize} = application:get_env(?APP_NAME, upload_read_size),

    case cowboy_req:part(Req) of
        {ok, Headers, Req2} ->
            case cow_multipart:form_data(Headers) of<commit_msg>VFS-2189 Close connection after file upload failure<commit_after>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C): 2016 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% This module processes file upload requests originating from GUI.
%%% Exposes an API required by lib ResumableJS that is used on client side.
%%% @end
%%%-------------------------------------------------------------------
-module(upload_handler).
-author("<NAME>").
-behaviour(cowboy_http_handler).

-include("global_definitions.hrl").
-include_lib("ctool/include/logging.hrl").

% Key of in-memory mapping of uploads kept in session.
-define(UPLOAD_MAP, upload_map).

% How many tries of creating a unique filename before failure.
-define(MAX_UNIQUE_FILENAME_COUNTER, 20).

% Maximum time a process will be waiting for chunk 1 to generate file handle.
-define(MAX_WAIT_FOR_FILE_HANDLE, 30000).

% Interval between retries to resolve file handle.
-define(INTERVAL_WAIT_FOR_FILE_HANDLE, 100).

%% Cowboy API
-export([init/3, handle/2, terminate/3]).
%% API
-export([upload_map_insert/2, upload_map_delete/1]).
-export([upload_map_lookup/1, upload_map_lookup/2]).
-export([clean_upload_map/0]).

%% ====================================================================
%% API functions
%% ====================================================================


%%--------------------------------------------------------------------
%% @doc
%% Inserts a UploadId - FileId pair into upload map.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_insert(UploadId :: term(), FileId :: term()) -> ok.
upload_map_insert(UploadId, FileId) ->
    Map = g_session:get_value(?UPLOAD_MAP, #{}),
    NewMap = maps:put(UploadId, FileId, Map),
    g_session:put_value(?UPLOAD_MAP, NewMap).


%%--------------------------------------------------------------------
%% @doc
%% Retrieves a FileId from upload map by UploadId.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_lookup(UploadId :: term()) -> FileId :: term().
upload_map_lookup(UploadId) ->
    upload_map_lookup(UploadId, undefined).


%%--------------------------------------------------------------------
%% @doc
%% Retrieves a FileId from upload map by UploadId or returns a default value.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_lookup(UploadId :: term(), Default :: term()) ->
    FileId :: term().
upload_map_lookup(UploadId, Default) ->
    Map = g_session:get_value(?UPLOAD_MAP, #{}),
    maps:get(UploadId, Map, Default).


%%--------------------------------------------------------------------
%% @doc
%% Deletes a UploadId - FileId pair from upload map.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec upload_map_delete(UploadId :: term()) -> ok.
upload_map_delete(UploadId) ->
    Map = g_session:get_value(?UPLOAD_MAP, #{}),
    NewMap = maps:remove(UploadId, Map),
    g_session:put_value(?UPLOAD_MAP, NewMap).


%%--------------------------------------------------------------------
%% @doc
%% Cleans all upload-related mappings in session memory.
%% Upload map is a map in session memory dedicated for handling chunked
%% file uploads.
%% @todo Should be redesigned in VFS-1815.
%% @end
%%--------------------------------------------------------------------
-spec clean_upload_map() -> ok.
clean_upload_map() ->
    g_session:put_value(?UPLOAD_MAP, #{}).


%% ====================================================================
%% Cowboy API functions
%% ====================================================================

%%--------------------------------------------------------------------
%% @doc
%% Cowboy handler callback.
%% @end
%%--------------------------------------------------------------------
-spec init({TransportName :: atom(), ProtocolName :: http},
    Req :: cowboy_req:req(), Opts :: any()) ->
    {ok, cowboy_req:req(), []}.
init(_Type, Req, _Opts) ->
    {ok, Req, []}.


%%--------------------------------------------------------------------
%% @doc
%% Handles an upload request.
%% @end
%%--------------------------------------------------------------------
-spec handle(term(), term()) -> {ok, cowboy_req:req(), term()}.
handle(Req, State) ->
    NewReq = handle_http_upload(Req),
    {ok, NewReq, State}.


%%--------------------------------------------------------------------
%% @doc
%% Cowboy handler callback, no cleanup needed
%% @end
%%--------------------------------------------------------------------
-spec terminate(term(), term(), term()) -> ok.
terminate(_Reason, _Req, _State) ->
    ok.


%% ====================================================================
%% Internal functions
%% ====================================================================

%%--------------------------------------------------------------------
%% @private
%% @doc
%% Asserts the validity of multipart POST request and proceeds with
%% parsing or returns an error. Returns list of parsed filed values and
%% file body.
%% @end
%%--------------------------------------------------------------------
-spec handle_http_upload(Req :: cowboy_req:req()) -> cowboy_req:req().
handle_http_upload(Req) ->
    % Try to retrieve user's session
    InitSession =
        try
            g_ctx:init(Req, false)
        catch _:_ ->
            % Error logging is done inside g_ctx:init
            error
        end,
    case InitSession of
        error ->
            g_ctx:reply(500, [{<<"connection">>, <<"close">>}], <<"">>);
        ok ->
            try
                NewReq = multipart(Req, []),
                g_ctx:set_cowboy_req(NewReq),
                g_ctx:reply(200, [], <<"">>)
            catch
                throw:{missing_param, _} ->
                    g_ctx:reply(500, [{<<"connection">>, <<"close">>}], <<"">>);
                throw:stream_file_error ->
                    g_ctx:reply(500, [{<<"connection">>, <<"close">>}], <<"">>);
                Type:Message ->
                    ?error_stacktrace("Error while processing file upload "
                    "from user ~p - ~p:~p",
                        [g_session:get_user_id(), Type, Message]),
                    % @todo VFS-1815 for now return 500,
                    % because retries are not stable
%%                    % Return 204 - resumable will retry the upload
%%                    g_ctx:reply(204, [], <<"">>)
                    g_ctx:reply(500, [{<<"connection">>, <<"close">>}], <<"">>)
            end
    end,
    g_ctx:finish().


%%--------------------------------------------------------------------
%% @private
%% @doc
%% Parses a multipart HTTP form.
%% @end
%%--------------------------------------------------------------------
-spec multipart(Req :: cowboy_req:req(), Params :: proplists:proplist()) ->
    cowboy_req:req().
multipart(Req, Params) ->
    {ok, UploadWriteSize} = application:get_env(?APP_NAME, upload_write_size),
    {ok, UploadReadTimeout} = application:get_env(?APP_NAME, upload_read_timeout),
    {ok, UploadReadSize} = application:get_env(?APP_NAME, upload_read_size),

    case cowboy_req:part(Req) of
        {ok, Headers, Req2} ->
            case cow_multipart:form_data(Headers) of
{'fraction_non_alphanumeric': 0.36387235659111056, 'fraction_numerical': 0.009370647081170065, 'mean_word_length': 1.6430722891566265, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4211843', 'n_tokens_mistral': 4149, 'n_tokens_neox': 3924, 'n_words': 1055}