<commit_before>%%%--------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2019 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%--------------------------------------------------------------------
%%% @doc
%%% This module is responsible for storing and distribution of static GUI files.
%%%
%%% GUI path under which files are stored consists of GUI prefix and GUI id.
%%% GUI prefix is gui prefix of service or harvester:
%%%     * ozw
%%%     * opw
%%%     * onp
%%%     * hrv
%%% GUI id is equal to id of corresponding service or harvester.
%%%
%%% In order to reuse the existing GUI packages, each GUI path is linked to the
%%% proper GUI version, denoted by package hash (SHA256), using symbolic links e.g.
%%%
%%%     /opw/9ba150771ad7b01366d17d89f19b56af  ->
%%%         /opw/4fc98577cee89c3dfb7817b11c0027ec3084f8ba455a162c9038ed568b9d3b7d
%%%
%%%     /opw/84b7331f7c230541d4177c23dd901fd1  ->
%%%         /opw/4fc98577cee89c3dfb7817b11c0027ec3084f8ba455a162c9038ed568b9d3b7d
%%%
%%% Special ./i file in every GUI path is an alias (symbolic link) for index.html:
%%%
%%%     /ozw/74afc09f584276186894b82caf466886/i  ->
%%%         /ozw/74afc09f584276186894b82caf466886/index.html
%%% @end
%%%--------------------------------------------------------------------
-module(gui_static).
-author("<NAME>").

-include("datastore/oz_datastore_models.hrl").
-include("http/gui_paths.hrl").
-include_lib("ctool/include/errors.hrl").
-include_lib("ctool/include/logging.hrl").
-include_lib("ctool/include/global_definitions.hrl").

-type gui_id() :: od_cluster:id() | od_harvester:id().
-export_type([gui_id/0]).

%% API
-export([deploy_package/3, deploy_package/4, ensure_package/4]).
-export([deploy_default_harvester_package/1, link_default_harvester_gui/1]).
-export([link_gui/3, link_gui/4]).
-export([unlink_gui/2, unlink_gui/3]).
-export([gui_exists/2]).
-export([remove_unused_packages/1, remove_unused_packages/2]).
-export([routes/0]).
-export([oz_worker_gui_path/1]).
-export([mimetype/1]).

-define(GUI_TYPES, [?OZ_WORKER_GUI, ?OP_WORKER_GUI, ?ONEPANEL_GUI, ?HARVESTER_GUI]).
% Only packages older than that will be cleaned (in seconds)
-define(CLEANING_AGE_THRESHOLD, 86400). % 1 day

-define(GUI_VERIFICATION_ENABLED, oz_worker:get_env(gui_package_verification, true)).
-define(HRV_GUI_VERIFICATION_ENABLED, oz_worker:get_env(harvester_gui_package_verification, true)).

-define(GUI_STATIC_ROOT, oz_worker:get_env(gui_static_root)).
-define(CUSTOM_STATIC_ROOT, oz_worker:get_env(gui_custom_static_root)).
-define(LEGACY_CUSTOM_STATIC_ROOT, oz_worker:get_env(legacy_gui_custom_static_root)).

-define(CLUSTER_NODES, consistent_hashing:get_all_nodes()).

-define(CRITICAL_SECTION(LockId, Fun), critical_section:run({gui_static, LockId}, Fun)).

-define(DEFAULT_HARVESTER_GUI_HASH, <<"default">>).

%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc
%% @equiv deploy_package(GuiType, ReleaseVsn, PackagePath, true)
%% @end
%%--------------------------------------------------------------------
-spec deploy_package(onedata:gui(), onedata:release_version(), file:name_all()) ->
    {ok, onedata:gui_hash()} | ?ERROR_BAD_GUI_PACKAGE |
    ?ERROR_GUI_PACKAGE_TOO_LARGE | ?ERROR_GUI_PACKAGE_UNVERIFIED(onedata:gui_hash()).
deploy_package(GuiType, ReleaseVsn, PackagePath) ->
    deploy_package(GuiType, ReleaseVsn, PackagePath, true).


%%--------------------------------------------------------------------
%% @doc
%% Reads given GUI package, and upon success, deploys the GUI package under given
%% GUI prefix on all cluster nodes. VerifyGuiHash flag decides if the GUI package
%% should be verified against known SHA256 checksums.
%% @end
%%--------------------------------------------------------------------
-spec deploy_package(onedata:gui(), onedata:release_version(), file:name_all(), VerifyGuiHash :: boolean()) ->
    {ok, onedata:gui_hash()} | ?ERROR_BAD_GUI_PACKAGE |
    ?ERROR_GUI_PACKAGE_TOO_LARGE | ?ERROR_GUI_PACKAGE_UNVERIFIED(onedata:gui_hash()).
deploy_package(GuiType, ReleaseVsn, PackagePath, VerifyGuiHash) ->
    case gui:read_package(PackagePath) of
        {ok, _, PackageBin} ->
            {ok, GuiHash} = gui:package_hash({binary, PackageBin}),
            case (not VerifyGuiHash) orelse verify_gui_hash(GuiType, ReleaseVsn, GuiHash) of
                true ->
                    ensure_package(GuiType, PackageBin, GuiHash),
                    {ok, GuiHash};
                false ->
                    ?ERROR_GUI_PACKAGE_UNVERIFIED(GuiHash)
            end;
        {error, _} = Error ->
            Error
    end.


%%--------------------------------------------------------------------
%% @doc
%% Reads given GUI package, and upon success, deploys the GUI package under 
%% given GUI prefix on all cluster nodes. 
%% Deployed package can be later linked using link_default_harvester_gui/1.
%% @end
%%--------------------------------------------------------------------
-spec deploy_default_harvester_package(file:name_all()) ->
    {ok, onedata:gui_hash()} | ?ERROR_BAD_GUI_PACKAGE | ?ERROR_GUI_PACKAGE_TOO_LARGE.
deploy_default_harvester_package(PackagePath) ->
    case gui:read_package(PackagePath) of
        {ok, _, PackageBin} -> ensure_package(?HARVESTER_GUI, PackageBin, ?DEFAULT_HARVESTER_GUI_HASH);
        {error, _} = Error -> Error
    end.


%%--------------------------------------------------------------------
%% @private
%% @doc
%% Deploys a GUI package under given GUI prefix on all cluster nodes. GuiHash
%% must be provided manually. The operation is skipped if the GUI package
%% already exists on all cluster nodes.
%% @end
%%--------------------------------------------------------------------
-spec ensure_package(onedata:gui(), PackageBin :: binary(), onedata:gui_hash()) ->
    ok.
ensure_package(GuiType, PackageBin, GuiHash) ->
    ?CRITICAL_SECTION(GuiHash, fun() ->
        case gui_exists_unsafe(GuiType, GuiHash) of
            true -> ok;
            false -> ensure_package(on_cluster, GuiType, PackageBin, GuiHash)
        end
    end).


%%--------------------------------------------------------------------
%% @doc
%% Deploys a GUI package under given  GUI prefix.
%%
%% NOTE: This function must be run in a critical section to avoid race conditions.
%%
%% Has two modes:
%%  on_cluster - performs the operation on all cluster nodes
%%  on_node - performs the operation only on the local node
%% @end
%%--------------------------------------------------------------------
-spec ensure_package(on_cluster | on_node, onedata:gui(), PackageBin :: binary(),
    onedata:gui_hash()) -> ok.
ensure_package(on_cluster, GuiType, PackageBin, GuiHash) ->
    lists:foreach(fun(Node) ->
        ok = rpc:call(Node, ?MODULE, ensure_package, [on_node, GuiType, PackageBin, GuiHash])
    end, ?CLUSTER_NODES);

ensure_package(on_node, GuiType, PackageBin, GuiHash) ->
    ?info("Deploying GUI package: ~s", [gui_path(GuiType, GuiHash)]),
    TempDir = mochitemp:mkdtemp(),
    {ok, ExtractedPackagePath} = gui:extract_package({binary, PackageBin}, TempDir),
    PackageStaticRoot = static_root(GuiType, GuiHash),
    ok = file_utils:recursive_del(PackageStaticRoot),
    ok = file_utils:move(ExtractedPackagePath, PackageStaticRoot),
    mochitemp:rmtempdir(TempDir).


%%--------------------------------------------------------------------
%% @doc
%% @equiv link_gui(on_cluster, GuiType, GuiId, GuiHash).
%% @end
%%--------------------------------------------------------------------
-spec link_gui(onedata:gui(), gui_id(), onedata:gui_hash()) -> ok.
link_gui(GuiType, GuiId, GuiHash) ->
    link_gui(on_cluster, GuiType, GuiId, GuiHash),
    schedule_removal_of_unused_packages().


%%--------------------------------------------------------------------
%% @doc
%% @equiv link_gui(?HARVESTER_GUI, GuiId, ?DEFAULT_HARVESTER_GUI_HASH).
%% @end
%%--------------------------------------------------------------------
-spec link_default_harvester_gui(od_harvester:id()) -> ok.
link_default_harvester_gui(HarvesterId) ->
    link_gui(?HARVESTER_GUI, HarvesterId, ?DEFAULT_HARVESTER_GUI_HASH).
<commit_msg>VFS-6544 Added removal of previous default harvester gui upon deployment<commit_after>%%%--------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2019 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%--------------------------------------------------------------------
%%% @doc
%%% This module is responsible for storing and distribution of static GUI files.
%%%
%%% GUI path under which files are stored consists of GUI prefix and GUI id.
%%% GUI prefix is gui prefix of service or harvester:
%%%     * ozw
%%%     * opw
%%%     * onp
%%%     * hrv
%%% GUI id is equal to id of corresponding service or harvester.
%%%
%%% In order to reuse the existing GUI packages, each GUI path is linked to the
%%% proper GUI version, denoted by package hash (SHA256), using symbolic links e.g.
%%%
%%%     /opw/9ba150771ad7b01366d17d89f19b56af  ->
%%%         /opw/4fc98577cee89c3dfb7817b11c0027ec3084f8ba455a162c9038ed568b9d3b7d
%%%
%%%     /opw/84b7331f7c230541d4177c23dd901fd1  ->
%%%         /opw/4fc98577cee89c3dfb7817b11c0027ec3084f8ba455a162c9038ed568b9d3b7d
%%%
%%% Special ./i file in every GUI path is an alias (symbolic link) for index.html:
%%%
%%%     /ozw/74afc09f584276186894b82caf466886/i  ->
%%%         /ozw/74afc09f584276186894b82caf466886/index.html
%%% @end
%%%--------------------------------------------------------------------
-module(gui_static).
-author("<NAME>").

-include("datastore/oz_datastore_models.hrl").
-include("http/gui_paths.hrl").
-include_lib("ctool/include/errors.hrl").
-include_lib("ctool/include/logging.hrl").
-include_lib("ctool/include/global_definitions.hrl").

-type gui_id() :: od_cluster:id() | od_harvester:id().
-export_type([gui_id/0]).

%% API
-export([deploy_package/3, deploy_package/4, put_package/4]).
-export([deploy_default_harvester_package/1, link_default_harvester_gui/1]).
-export([link_gui/3, link_gui/4]).
-export([unlink_gui/2, unlink_gui/3]).
-export([gui_exists/2]).
-export([remove_unused_packages/1, remove_unused_packages/2]).
-export([routes/0]).
-export([oz_worker_gui_path/1]).
-export([mimetype/1]).

-define(GUI_TYPES, [?OZ_WORKER_GUI, ?OP_WORKER_GUI, ?ONEPANEL_GUI, ?HARVESTER_GUI]).
% Only packages older than that will be cleaned (in seconds)
-define(CLEANING_AGE_THRESHOLD, 86400). % 1 day

-define(GUI_VERIFICATION_ENABLED, oz_worker:get_env(gui_package_verification, true)).
-define(HRV_GUI_VERIFICATION_ENABLED, oz_worker:get_env(harvester_gui_package_verification, true)).

-define(GUI_STATIC_ROOT, oz_worker:get_env(gui_static_root)).
-define(CUSTOM_STATIC_ROOT, oz_worker:get_env(gui_custom_static_root)).
-define(LEGACY_CUSTOM_STATIC_ROOT, oz_worker:get_env(legacy_gui_custom_static_root)).

-define(CLUSTER_NODES, consistent_hashing:get_all_nodes()).

-define(CRITICAL_SECTION(LockId, Fun), critical_section:run({gui_static, LockId}, Fun)).

-define(DEFAULT_HARVESTER_GUI_HASH, <<"default">>).

%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc
%% @equiv deploy_package(GuiType, ReleaseVsn, PackagePath, true)
%% @end
%%--------------------------------------------------------------------
-spec deploy_package(onedata:gui(), onedata:release_version(), file:name_all()) ->
    {ok, onedata:gui_hash()} | ?ERROR_BAD_GUI_PACKAGE |
    ?ERROR_GUI_PACKAGE_TOO_LARGE | ?ERROR_GUI_PACKAGE_UNVERIFIED(onedata:gui_hash()).
deploy_package(GuiType, ReleaseVsn, PackagePath) ->
    deploy_package(GuiType, ReleaseVsn, PackagePath, true).


%%--------------------------------------------------------------------
%% @doc
%% Reads given GUI package, and upon success, deploys the GUI package under given
%% GUI prefix on all cluster nodes. VerifyGuiHash flag decides if the GUI package
%% should be verified against known SHA256 checksums.
%% @end
%%--------------------------------------------------------------------
-spec deploy_package(onedata:gui(), onedata:release_version(), file:name_all(), VerifyGuiHash :: boolean()) ->
    {ok, onedata:gui_hash()} | ?ERROR_BAD_GUI_PACKAGE |
    ?ERROR_GUI_PACKAGE_TOO_LARGE | ?ERROR_GUI_PACKAGE_UNVERIFIED(onedata:gui_hash()).
deploy_package(GuiType, ReleaseVsn, PackagePath, VerifyGuiHash) ->
    case gui:read_package(PackagePath) of
        {ok, _, PackageBin} ->
            {ok, GuiHash} = gui:package_hash({binary, PackageBin}),
            case (not VerifyGuiHash) orelse verify_gui_hash(GuiType, ReleaseVsn, GuiHash) of
                true ->
                    ensure_package(GuiType, PackageBin, GuiHash),
                    {ok, GuiHash};
                false ->
                    ?ERROR_GUI_PACKAGE_UNVERIFIED(GuiHash)
            end;
        {error, _} = Error ->
            Error
    end.


%%--------------------------------------------------------------------
%% @doc
%% Reads given GUI package, and upon success, deploys the GUI package under 
%% given GUI prefix on all cluster nodes. 
%% Removes previously deployed default package if it exists.
%% Deployed package can be later linked using link_default_harvester_gui/1.
%% @end
%%--------------------------------------------------------------------
-spec deploy_default_harvester_package(file:name_all()) ->
    ok | ?ERROR_BAD_GUI_PACKAGE | ?ERROR_GUI_PACKAGE_TOO_LARGE.
deploy_default_harvester_package(PackagePath) ->
    case gui:read_package(PackagePath) of
        {ok, _, PackageBin} ->
            ?CRITICAL_SECTION(?DEFAULT_HARVESTER_GUI_HASH, fun() ->
                put_package(on_cluster, ?HARVESTER_GUI, PackageBin, ?DEFAULT_HARVESTER_GUI_HASH)
            end);
        {error, _} = Error -> Error
    end.


%%--------------------------------------------------------------------
%% @private
%% @doc
%% Deploys a GUI package under given GUI prefix on all cluster nodes. GuiHash
%% must be provided manually. The operation is skipped if the GUI package
%% already exists on all cluster nodes.
%% @end
%%--------------------------------------------------------------------
-spec ensure_package(onedata:gui(), PackageBin :: binary(), onedata:gui_hash()) ->
    ok.
ensure_package(GuiType, PackageBin, GuiHash) ->
    ?CRITICAL_SECTION(GuiHash, fun() ->
        case gui_exists_unsafe(GuiType, GuiHash) of
            true -> ok;
            false -> put_package(on_cluster, GuiType, PackageBin, GuiHash)
        end
    end).


%%--------------------------------------------------------------------
%% @doc
%% Extracts and places given GUI package under the path corresponding to the GUI 
%% type and hash. Overwrites previous package (if any).
%%
%% NOTE: This function must be run in a critical section to avoid race conditions.
%%
%% Has two modes:
%%  on_cluster - performs the operation on all cluster nodes
%%  on_node - performs the operation only on the local node
%% @end
%%--------------------------------------------------------------------
-spec put_package(on_cluster | on_node, onedata:gui(), PackageBin :: binary(),
    onedata:gui_hash()) -> ok.
put_package(on_cluster, GuiType, PackageBin, GuiHash) ->
    lists:foreach(fun(Node) ->
        ok = rpc:call(Node, ?MODULE, put_package, [on_node, GuiType, PackageBin, GuiHash])
    end, ?CLUSTER_NODES);

put_package(on_node, GuiType, PackageBin, GuiHash) ->
    ?info("Deploying GUI package: ~s", [gui_path(GuiType, GuiHash)]),
    TempDir = mochitemp:mkdtemp(),
    {ok, ExtractedPackagePath} = gui:extract_package({binary, PackageBin}, TempDir),
    PackageStaticRoot = static_root(GuiType, GuiHash),
    ok = file_utils:recursive_del(PackageStaticRoot),
    ok = file_utils:move(ExtractedPackagePath, PackageStaticRoot),
    mochitemp:rmtempdir(TempDir).


%%--------------------------------------------------------------------
%% @doc
%% @equiv link_gui(on_cluster, GuiType, GuiId, GuiHash).
%% @end
%%--------------------------------------------------------------------
-spec link_gui(onedata:gui(), gui_id(), onedata:gui_hash()) -> ok.
link_gui(GuiType, GuiId, GuiHash) ->
    link_gui(on_cluster, GuiType, GuiId, GuiHash),
    schedule_removal_of_unused_packages().


%%--------------------------------------------------------------------
%% @doc
%% @equiv link_gui(?HARVESTER_GUI, GuiId, ?DEFAULT_HARVESTER_GUI_HASH).
%% @end
%%--------------------------------------------------------------------
-spec link_default_harvester_gui(od_harvester:id()) -> ok.
link_default_harvester_gui(HarvesterId) ->
    link_gui(?HARVESTER_GUI, HarvesterId, ?DEFAULT_HARVESTER_GUI_HASH).

{'fraction_non_alphanumeric': 0.26562683931724546, 'fraction_numerical': 0.024367274867569157, 'mean_word_length': 2.469675311415152, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4303993', 'n_tokens_mistral': 5494, 'n_tokens_neox': 5166, 'n_words': 1081}