<commit_before>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%      http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
% A module for retrieving & configuring per-server storage paths,
% storage quotas, mem quotas, etc.
%
-module(ns_storage_conf).

-include_lib("eunit/include/eunit.hrl").

-include("ns_common.hrl").

-export([memory_quota/1, change_memory_quota/2,
         setup_disk_storage_conf/2,
         storage_conf/1, storage_conf_from_node_status/1, add_storage/4, remove_storage/2,
         local_bucket_disk_usage/1,
         this_node_dbdir/0, this_node_ixdir/0, this_node_logdir/0,
         this_node_bucket_dbdir/1,
         delete_databases_and_files/1, delete_unused_buckets_db_files/0,
         bucket_databases/1]).

-export([node_storage_info/1, cluster_storage_info/0, nodes_storage_info/1]).

-export([allowed_node_quota_range/1, allowed_node_quota_range/0,
         default_memory_quota/1,
         allowed_node_quota_range_for_joined_nodes/0,
         this_node_memory_data/0]).

-export([extract_disk_stats_for_path/2]).


memory_quota(Node) ->
    memory_quota(Node, ns_config:get()).

memory_quota(_Node, Config) ->
    {value, RV} = ns_config:search(Config, memory_quota),
    RV.


-spec this_node_bucket_dirs(BucketName :: string()) -> [string()].
this_node_bucket_dirs(BucketName) ->
    {ok, DBDir} = this_node_dbdir(),
    {ok, IxDir} = this_node_ixdir(),

    [filename:join(DBDir, BucketName),
     filename:join(IxDir, "." ++ BucketName)].

couch_storage_path(Field) ->
    try cb_config_couch_sync:get_db_and_ix_paths() of
        PList ->
            {Field, RV} = lists:keyfind(Field, 1, PList),
            {ok, RV}
    catch T:E ->
            ?log_debug("Failed to get couch storage config field: ~p due to ~p:~p", [Field, T, E]),
            {error, iolist_to_binary(io_lib:format("couch config access failed: ~p:~p", [T, E]))}
    end.

-spec this_node_dbdir() -> {ok, string()} | {error, binary()}.
this_node_dbdir() ->
    couch_storage_path(db_path).

-spec this_node_bucket_dbdir(bucket_name()) -> {ok, string()}.
this_node_bucket_dbdir(BucketName) ->
    {ok, DBDir} = ns_storage_conf:this_node_dbdir(),
    DBSubDir = filename:join(DBDir, BucketName),
    {ok, DBSubDir}.

-spec this_node_ixdir() -> {ok, string()} | {error, binary()}.
this_node_ixdir() ->
    couch_storage_path(index_path).

-spec this_node_logdir() -> {ok, string()} | {error, any()}.
this_node_logdir() ->
    logdir(ns_config:get(), node()).

-spec logdir(any(), atom()) -> {ok, string()} | {error, any()}.
logdir(Config, Node) ->
    read_path_from_conf(Config, Node, ns_log, filename).

%% @doc read a path from the configuration, following symlinks
-spec read_path_from_conf(any(), atom(), atom(), atom()) ->
    {ok, string()} | {error, any()}.
read_path_from_conf(Config, Node, Key, SubKey) ->
    {value, PropList} = ns_config:search_node(Node, Config, Key),
    case proplists:get_value(SubKey, PropList) of
        undefined ->
            {error, undefined};
        DBDir ->
            {ok, Base} = file:get_cwd(),
            case misc:realpath(DBDir, Base) of
                {error, Atom, _, _} -> {error, Atom};
                {ok, _} = X -> X
            end
    end.


change_memory_quota(_Node, NewMemQuotaMB) when is_integer(NewMemQuotaMB) ->
    ns_config:set(memory_quota, NewMemQuotaMB).

ensure_dir(Path) ->
    filelib:ensure_dir(Path),
    case file:make_dir(Path) of
        ok -> ok;
        {error, eexist} ->
            TouchPath = filename:join(Path, ".touch"),
            case file:write_file(TouchPath, <<"">>) of
                ok ->
                    file:delete(TouchPath),
                    ok;
                _ -> error
            end;
        _ -> error
    end.

ensure_dirs([]) ->
    ok;
ensure_dirs([Path | Rest]) ->
    case ensure_dir(Path) of
        ok ->
            ensure_dirs(Rest);
        X -> X
    end.

%% @doc sets db and index path of this node.
%%
%% NOTE: ns_server restart is required to make this fully effective.
-spec setup_disk_storage_conf(DbPath::string(), IxDir::string()) -> ok | {errors, [Msg :: binary()]}.
setup_disk_storage_conf(DbPath, IxPath) ->
    [{db_path, CurrentDbDir},
     {index_path, CurrentIxDir}] = lists:sort(cb_config_couch_sync:get_db_and_ix_paths()),

    NewDbDir = misc:absname(DbPath),
    NewIxDir = misc:absname(IxPath),

    case NewDbDir =/= CurrentDbDir orelse NewIxDir =/= CurrentIxDir of
        true ->
            ale:info(?USER_LOGGER, "Setting database directory path to ~s and index directory path to ~s", [NewDbDir, NewIxDir]),
            case ensure_dirs([NewDbDir, NewIxDir]) of
                ok ->
                    RV =
                        case NewDbDir =:= CurrentDbDir of
                            true ->
                                ok;
                            false ->
                                ?log_info("Removing all the buckets because "
                                          "database path has changed (old database path ~s)",
                                          [CurrentDbDir]),
                                [ns_orchestrator:delete_bucket(Bucket)
                                 || {Bucket, _} <- ns_bucket:get_buckets()],

                                ?log_info("Removing all unused database files"),
                                case delete_unused_buckets_db_files() of
                                    ok ->
                                        ok;
                                    Error ->
                                        Msg = iolist_to_binary(
                                                io_lib:format("Could not delete unused database files: ~p",
                                                              [Error])),
                                        {errors, [Msg]}
                                end
                        end,

                    case RV of
                        ok ->
                            cb_config_couch_sync:set_db_and_ix_paths(NewDbDir, NewIxDir);
                        _ ->
                            RV
                    end;
                error ->
                    {errors, [<<"Could not set the storage path. It must be a directory writable by 'couchbase' user.">>]}
            end;
        _ ->
            ok
    end.

local_bucket_disk_usage(BucketName) ->
    BucketDirs = this_node_bucket_dirs(BucketName),
    lists:sum([misc:dir_size(Dir) || Dir <- BucketDirs]).

% Returns a proplist of lists of proplists.
%
% A quotaMb of none means no quota. Disks can get full, disappear,
% etc, so non-ok state is used to signal issues.
%
% NOTE: in current implementation node disk quota is supported and
% state is always ok
%
% NOTE: current node supports only single storage path and does not
% support dedicated ssd (versus hdd) path
%
% NOTE: 1.7/1.8 nodes will not have storage conf returned in current
% implementation.
%
% [{ssd, []},
%  {hdd, [[{path, /some/nice/disk/path}, {quotaMb, 1234}, {state, ok}],
%         [{path", /another/good/disk/path}, {quotaMb, 5678}, {state, ok}]]}]
%
storage_conf(Node) ->
    NodeStatus = ns_doctor:get_node(Node),
    storage_conf_from_node_status(NodeStatus).

storage_conf_from_node_status(NodeStatus) ->
    StorageConf = proplists:get_value(node_storage_conf, NodeStatus, []),
    HDDInfo = case proplists:get_value(db_path, StorageConf) of
                  undefined -> [];
                  DBDir ->
                      [{path, DBDir},
                       {index_path, proplists:get_value(index_path, StorageConf, DBDir)},
                       {quotaMb, none},
                       {state, ok}]
              end,
    [{ssd, []},
     {hdd, [HDDInfo]}].

% Quota is an integer or atom none.
% Kind is atom ssd or hdd.
%
add_storage(_Node, "", _Kind, _Quota) ->
    {error, invalid_path};

add_storage(_Node, _Path, _Kind, _Quota) ->
    % TODO.
    ok.

remove_storage(_Node, _Path) ->
    % TODO.
    {error, todo}.

node_storage_info(Node) ->
    case dict:find(Node, ns_doctor:get_nodes()) of
        {ok, NodeInfo} ->
            extract_node_storage_info(NodeInfo, Node);
        _ -> []
    end.

extract_node_storage_info(NodeInfo, Node) ->
    extract_node_storage_info(NodeInfo, Node, ns_config:get()).

extract_node_storage_info(NodeInfo, Node, Config) ->
    {RAMTotal, RAMUsed, _} = proplists:get_value(memory_data, NodeInfo),
    DiskStats = proplists:get_value(disk_data, NodeInfo),
    StorageConf = proplists:get_value(node_storage_conf, NodeInfo, []),
    DiskPaths = [X || {PropName, X} <- StorageConf,
                      PropName =:= db_path orelse PropName =:= index_path],
    case memory_quota(Node, Config) of
        MemQuotaMB when is_integer(MemQuotaMB) ->
            {DiskTotal, DiskUsed} = extract_disk_totals(DiskPaths, DiskStats),
            [{ram, [{total, RAMTotal},
                    {quotaTotal, MemQuotaMB * 1048576},
                    {used, RAMUsed},
                    {free, 0} % not used
                   ]},
             {hdd, [{total, DiskTotal},
                    {quotaTotal, DiskTotal},
                    {used, DiskUsed},
                    {free, DiskTotal - DiskUsed}
                   ]}];
        _ -> []
    end.

-spec extract_disk_totals(list(), list()) -> {integer(), integer()}.
extract_disk_totals(DiskPaths, DiskStats) ->

    F = fun (Path, {UsedMounts, ATotal, AUsed} = Tuple) ->
                case extract_disk_stats_for_path(DiskStats, Path) of
                    none -> Tuple;
                    {ok, {MPoint, KBytesTotal, Cap}} ->
                        case lists:member(MPoint, UsedMounts) of
                            true -> Tuple;
                            false ->
                                Total = KBytesTotal * 1024,
                                Used = (Total * Cap) div 100,
                                {[MPoint | UsedMounts], ATotal + Total,
                                 AUsed + Used}
                        end
                end
        end,
    {_UsedMounts, DiskTotal, DiskUsed} = lists:foldl(F, {[], 0, 0}, DiskPaths),
    {DiskTotal, DiskUsed}.

%% returns cluster_storage_info for subset of nodes
nodes_storage_info(NodeNames) ->
    NodesDict = ns_doctor:get_nodes(),
    NodesInfos = lists:foldl(fun (N, A) ->
                                     case dict:find(N, NodesDict) of
                                         {ok, V} -> [{N, V} | A];
                                         _ -> A
                                     end
                             end, [], NodeNames),
    do_cluster_storage_info(NodesInfos).

%% returns cluster storage info. This is aggregation of various
%% storage related metrics across active nodes of cluster.
%%
%% total - total amount of this resource (ram or hdd) in bytes
%%
%% free - amount of this resource free (ram or hdd) in bytes
%%
%% used - amount of this resource used (for any purpose (us, OS, other
%% processes)) in bytes
%%
%% quotaTotal - amount of quota for this resource across cluster
%% nodes. Note hdd quota is not really supported.
%%
%% quotaUsed - amount of quota already allocated
%%
%% usedByData - amount of this resource used by our data
cluster_storage_info() ->
    nodes_storage_info(ns_cluster_membership:active_nodes()).

extract_subprop(NodeInfos, Key, SubKey) ->
    [proplists:get_value(SubKey, proplists:get_value(Key, NodeInfo, [])) ||
     NodeInfo <- NodeInfos].

do_cluster_storage_info([]) -> [];
do_cluster_storage_info(NodeInfos) ->
    Config = ns_config:get(),
    AllBuckets = ns_bucket:get_buckets(Config),
    AllNodes = ordsets:intersection(lists:sort(ns_node_disco:nodes_actual_proper()),
                                    lists:sort(proplists:get_keys(NodeInfos))),
    AllNodesSize = length(AllNodes),
    RAMQuotaUsed = lists:foldl(fun ({_, BucketConfig}, RAMQuota) ->
                                       ns_bucket:raw_ram_quota(BucketConfig) * AllNodesSize + RAMQuota
                               end, 0, AllBuckets),
    StorageInfos = [extract_node_storage_info(NodeInfo, Node, Config)
                    || {Node, NodeInfo} <- NodeInfos],
    HddTotals = extract_subprop(StorageInfos, hdd, total),
    HddUsed = extract_subprop(StorageInfos, hdd, used),


    BucketsRAMUsage = lists:foldl(
                        fun ({_, Info}, RAMUsage) ->
                                case lists:keyfind(interesting_stats, 1, Info) of
                                    false ->
                                        RAMUsage;
                                    {_, InterestingStats} ->
                                        case lists:keyfind(mem_used, 1, InterestingStats) of
                                            false ->
                                                RAMUsage;
                                            {_, V} ->
                                                RAMUsage + V
                                        end
                                end
                        end, 0, NodeInfos),

    BucketsDiskUsage = lists:foldl(
                         fun ({_, Info}, DiskUsage) ->
                                 case lists:keyfind(interesting_stats, 1, Info) of
                                     false ->
                                         DiskUsage;
                                     {_, InterestingStats} ->
                                         V2 = case lists:keyfind(couch_docs_actual_disk_size, 1, InterestingStats) of
                                                  false ->
                                                      DiskUsage;
                                                  {_, X} ->
                                                      DiskUsage + X
                                              end,
                                         case lists:keyfind(couch_views_actual_disk_size, 1, InterestingStats) of
                                             false ->
                                                 V2;
                                             {_, V} ->
                                                 V2 + V
                                         end
                                 end
                         end, 0, NodeInfos),

    RAMUsed = erlang:max(lists:sum(extract_subprop(StorageInfos, ram, used)),
                         BucketsRAMUsage),
    HDDUsed = erlang:max(lists:sum(HddUsed),
                         BucketsDiskUsage),

    [{ram, [{total, lists:sum(extract_subprop(StorageInfos, ram, total))},
            {quotaTotal, lists:sum(extract_subprop(StorageInfos, ram,
                                                   quotaTotal))},
            {quotaUsed, RAMQuotaUsed},
            {used, RAMUsed},
            {usedByData, BucketsRAMUsage}
           ]},
     {hdd, [{total, lists:sum(HddTotals)},
            {quotaTotal, lists:sum(HddTotals)},
            {used, HDDUsed},
            {usedByData, BucketsDiskUsage},
            {free, lists:min(lists:zipwith(fun (A, B) -> A - B end,
                                           HddTotals, HddUsed)) * length(HddUsed)} % Minimum amount free on any node * number of nodes
           ]}].

extract_disk_stats_for_path_rec([], _Path) ->
    none;
extract_disk_stats_for_path_rec([{MountPoint0, _, _} = Info | Rest], Path) ->
    MountPoint = filename:join([MountPoint0]),  % normalize path. See filename:join docs
    MPath = case lists:reverse(MountPoint) of
                %% ends of '/'
                "/" ++ _ -> MountPoint;
                %% doesn't. Append it
                X -> lists:reverse("/" ++ X)
            end,
    case MPath =:= string:substr(Path, 1, length(MPath)) of
        true -> {ok, Info};
        _ -> extract_disk_stats_for_path_rec(Rest, Path)
    end.

%% Path0 must be an absolute path with all symlinks resolved.
extract_disk_stats_for_path(StatsList, Path0) ->
    Path = case filename:join([Path0]) of
               "/" -> "/";
               X -> X ++ "/"
           end,
    %% we sort by decreasing length so that first match is 'deepest'
    LessEqFn = fun (A,B) ->
                       length(element(1, A)) >= length(element(1, B))
               end,
    SortedList = lists:sort(LessEqFn, StatsList),
    extract_disk_stats_for_path_rec(SortedList, Path).

bucket_databases(Bucket) when is_list(Bucket) ->
    bucket_databases(list_to_binary(Bucket));
bucket_databases(Bucket) when is_binary(Bucket) ->
    couch_server:all_known_databases_with_prefix(iolist_to_binary([Bucket, $/])).

delete_couch_database(DB) ->
    RV = couch_server:delete(DB, []),
    ?log_info("Deleting database ~p: ~p~n", [DB, RV]),
    RV.

%% scan data directory for bucket names
bucket_names_from_disk() ->
    {ok, DbDir} = ns_storage_conf:this_node_dbdir(),
    Files = filelib:wildcard("*", DbDir),
    lists:foldl(
      fun (MaybeBucket, Acc) ->
              Path = filename:join(DbDir, MaybeBucket),
              case filelib:is_dir(Path) andalso
                  ns_bucket:is_valid_bucket_name(MaybeBucket) of
                  true ->
                      [MaybeBucket | Acc];
                  false ->
                      Acc
              end
      end, [], Files).

delete_disk_buckets_databases(Pred) ->
    Buckets = lists:filter(Pred, bucket_names_from_disk()),
    delete_disk_buckets_databases_loop(Pred, Buckets).

delete_disk_buckets_databases_loop(_Pred, []) ->
    ok;
delete_disk_buckets_databases_loop(Pred, [Bucket | Rest]) ->
    case delete_databases_and_files(Bucket) of
        ok ->
            delete_disk_buckets_databases_loop(Pred, Rest);
        Error ->
            Error
    end.


delete_databases_and_files(Bucket) ->
    AllDBs = bucket_databases(Bucket),
    MasterDB = iolist_to_binary([Bucket, <<"/master">>]),
    {MaybeMasterDb, RestDBs} = lists:partition(
                            fun (Name) ->
                                    Name =:= MasterDB
                            end, AllDBs),
    RV = case delete_databases_loop(MaybeMasterDb ++ RestDBs) of
             ok ->
                 ?log_info("Couch dbs are deleted. Proceeding with bucket directory"),
                 {ok, DbDir} = ns_storage_conf:this_node_dbdir(),
                 Path = filename:join(DbDir, Bucket),
                 case misc:rm_rf(Path) of
                     ok -> ok;
                     Error ->
                         ale:error(?USER_LOGGER, "Unable to rm -rf bucket database directory ~s~n~p", [Bucket, Error]),
                         Error
                 end;
             Error ->
                 ale:error(?USER_LOGGER, "Unable to delete some DBs for bucket ~s. Leaving bucket directory undeleted~n~p", [Bucket, Error]),
                 Error
         end,
    do_delete_bucket_indexes(Bucket),
    RV.

do_delete_bucket_indexes(Bucket) ->
    {ok, BaseIxDir} = this_node_ixdir(),
    IxDir = filename:join([BaseIxDir, "@indexes", Bucket]),
    case file:read_file_info(IxDir) of
        {error, enoent} ->
            ?log_debug("indexes directory doesn't exist already. fine.");
        _ ->
            ?log_debug("Going to delete indexes directory: ~p", [IxDir]),
            case misc:rm_rf(IxDir) of
                ok ->
                    ?log_debug("done deleting indexes directory");
                Error ->
                    ?log_debug("failed to delete indexes directory (~p). Will try to ignore", [Error])
            end
    end.

delete_databases_loop([]) ->
    ok;
delete_databases_loop([Db | Rest]) ->
    case delete_couch_database(Db) of
        ok ->
            delete_databases_loop(Rest);
        Error ->
            Error
    end.


%% deletes all databases files for buckets not defined for this node
%% note: this is called remotely
%%
%% it's named a bit differently from other functions here; but this function
%% is rpc called by older nodes; so we must keep this name unchanged
delete_unused_buckets_db_files() ->
    Config = ns_config:get(),
    BucketNames = ns_bucket:node_bucket_names(node(), ns_bucket:get_buckets(Config)),
    delete_disk_buckets_databases(
      fun (Bucket) ->
              RV = not(lists:member(Bucket, BucketNames)),
              case RV of
                  true ->
                      ale:info(?USER_LOGGER, "Deleting old data files of bucket ~p", [Bucket]);
                  _ ->
                      ok
              end,
              RV
      end).

-ifdef(EUNIT).
extract_disk_stats_for_path_test() ->
    DiskSupStats = [{"/",297994252,97},
             {"/lib/init/rw",1921120,1},
             {"/dev",10240,2},
             {"/dev/shm",1921120,0},
             {"/var/separate",1921120,0},
             {"/media/p2",9669472,81}],
    ?assertEqual({ok, {"/media/p2",9669472,81}},
                 extract_disk_stats_for_path(DiskSupStats,
                                             "/media/p2/mbdata")),
    ?assertEqual({ok, {"/", 297994252, 97}},
                 extract_disk_stats_for_path(DiskSupStats, "/")),
    ?assertEqual({ok, {"/", 297994252, 97}},
                 extract_disk_stats_for_path(DiskSupStats, "/lib/init")),
    ?assertEqual({ok, {"/dev", 10240, 2}},
                 extract_disk_stats_for_path(DiskSupStats, "/dev/sh")),
    ?assertEqual({ok, {"/dev", 10240, 2}},
                 extract_disk_stats_for_path(DiskSupStats, "/dev")).


-endif.

this_node_memory_data() ->
    case os:getenv("MEMBASE_RAM_MEGS") of
        false -> memsup:get_memory_data();
        X ->
            RAMBytes = list_to_integer(X) * 1048576,
            {RAMBytes, 0, 0}
    end.

allowed_node_quota_range() ->
    MemoryData = this_node_memory_data(),
    allowed_node_quota_range(MemoryData, 1024).

allowed_node_quota_range(MemoryData) ->
    allowed_node_quota_range(MemoryData, 1024).

%% when validating memory size versus cluster quota we use less strict
%% rules so that clusters upgraded from 1.6.0 are able to join
%% homogeneous nodes. See MB-2762
allowed_node_quota_range_for_joined_nodes() ->
    MemoryData = this_node_memory_data(),
    allowed_node_quota_range(MemoryData, 512).

allowed_node_quota_range(MemSupData, MinusMegs) ->
    {MaxMemoryBytes0, _, _} = MemSupData,
    MiB = 1048576,
    MinMemoryMB = 256,
    MaxMemoryMBPercent = (MaxMemoryBytes0 * 4) div (5 * MiB),
    MaxMemoryMB = lists:max([(MaxMemoryBytes0 div MiB) - MinusMegs,
                             MaxMemoryMBPercent]),
    QuotaErrorDetailsFun = fun () ->
                                   case MaxMemoryMB of
                                       MaxMemoryMBPercent ->
                                           io_lib:format(" Quota must be between 256 MB and ~w MB (80% of memory size).", [MaxMemoryMB]);
                                       _ ->
                                           io_lib:format(" Quota must be between 256 MB and ~w MB (memory size minus ~w MB).", [MaxMemoryMB, MinusMegs])
                                   end
                           end,
    {MinMemoryMB, MaxMemoryMB, QuotaErrorDetailsFun}.

default_memory_quota(MemSupData) ->
    {Min, Max, _} = allowed_node_quota_range(MemSupData),
    {MaxMemoryBytes0, _, _} = MemSupData,
    MiB = 1048576,
    Value = (MaxMemoryBytes0 * 3) div (5 * MiB),
    if Value > Max ->
            Max;
       Value < Min ->
            Min;
       true ->
            Value
    end.
<commit_msg>MB-7799 Use precomputed value in local_bucket_disk_usage.<commit_after>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%      http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
% A module for retrieving & configuring per-server storage paths,
% storage quotas, mem quotas, etc.
%
-module(ns_storage_conf).

-include_lib("eunit/include/eunit.hrl").

-include("ns_common.hrl").

-export([memory_quota/1, change_memory_quota/2,
         setup_disk_storage_conf/2,
         storage_conf/1, storage_conf_from_node_status/1, add_storage/4, remove_storage/2,
         local_bucket_disk_usage/1,
         this_node_dbdir/0, this_node_ixdir/0, this_node_logdir/0,
         this_node_bucket_dbdir/1,
         delete_databases_and_files/1, delete_unused_buckets_db_files/0,
         bucket_databases/1]).

-export([node_storage_info/1, cluster_storage_info/0, nodes_storage_info/1]).

-export([allowed_node_quota_range/1, allowed_node_quota_range/0,
         default_memory_quota/1,
         allowed_node_quota_range_for_joined_nodes/0,
         this_node_memory_data/0]).

-export([extract_disk_stats_for_path/2]).


memory_quota(Node) ->
    memory_quota(Node, ns_config:get()).

memory_quota(_Node, Config) ->
    {value, RV} = ns_config:search(Config, memory_quota),
    RV.

couch_storage_path(Field) ->
    try cb_config_couch_sync:get_db_and_ix_paths() of
        PList ->
            {Field, RV} = lists:keyfind(Field, 1, PList),
            {ok, RV}
    catch T:E ->
            ?log_debug("Failed to get couch storage config field: ~p due to ~p:~p", [Field, T, E]),
            {error, iolist_to_binary(io_lib:format("couch config access failed: ~p:~p", [T, E]))}
    end.

-spec this_node_dbdir() -> {ok, string()} | {error, binary()}.
this_node_dbdir() ->
    couch_storage_path(db_path).

-spec this_node_bucket_dbdir(bucket_name()) -> {ok, string()}.
this_node_bucket_dbdir(BucketName) ->
    {ok, DBDir} = ns_storage_conf:this_node_dbdir(),
    DBSubDir = filename:join(DBDir, BucketName),
    {ok, DBSubDir}.

-spec this_node_ixdir() -> {ok, string()} | {error, binary()}.
this_node_ixdir() ->
    couch_storage_path(index_path).

-spec this_node_logdir() -> {ok, string()} | {error, any()}.
this_node_logdir() ->
    logdir(ns_config:get(), node()).

-spec logdir(any(), atom()) -> {ok, string()} | {error, any()}.
logdir(Config, Node) ->
    read_path_from_conf(Config, Node, ns_log, filename).

%% @doc read a path from the configuration, following symlinks
-spec read_path_from_conf(any(), atom(), atom(), atom()) ->
    {ok, string()} | {error, any()}.
read_path_from_conf(Config, Node, Key, SubKey) ->
    {value, PropList} = ns_config:search_node(Node, Config, Key),
    case proplists:get_value(SubKey, PropList) of
        undefined ->
            {error, undefined};
        DBDir ->
            {ok, Base} = file:get_cwd(),
            case misc:realpath(DBDir, Base) of
                {error, Atom, _, _} -> {error, Atom};
                {ok, _} = X -> X
            end
    end.


change_memory_quota(_Node, NewMemQuotaMB) when is_integer(NewMemQuotaMB) ->
    ns_config:set(memory_quota, NewMemQuotaMB).

ensure_dir(Path) ->
    filelib:ensure_dir(Path),
    case file:make_dir(Path) of
        ok -> ok;
        {error, eexist} ->
            TouchPath = filename:join(Path, ".touch"),
            case file:write_file(TouchPath, <<"">>) of
                ok ->
                    file:delete(TouchPath),
                    ok;
                _ -> error
            end;
        _ -> error
    end.

ensure_dirs([]) ->
    ok;
ensure_dirs([Path | Rest]) ->
    case ensure_dir(Path) of
        ok ->
            ensure_dirs(Rest);
        X -> X
    end.

%% @doc sets db and index path of this node.
%%
%% NOTE: ns_server restart is required to make this fully effective.
-spec setup_disk_storage_conf(DbPath::string(), IxDir::string()) -> ok | {errors, [Msg :: binary()]}.
setup_disk_storage_conf(DbPath, IxPath) ->
    [{db_path, CurrentDbDir},
     {index_path, CurrentIxDir}] = lists:sort(cb_config_couch_sync:get_db_and_ix_paths()),

    NewDbDir = misc:absname(DbPath),
    NewIxDir = misc:absname(IxPath),

    case NewDbDir =/= CurrentDbDir orelse NewIxDir =/= CurrentIxDir of
        true ->
            ale:info(?USER_LOGGER, "Setting database directory path to ~s and index directory path to ~s", [NewDbDir, NewIxDir]),
            case ensure_dirs([NewDbDir, NewIxDir]) of
                ok ->
                    RV =
                        case NewDbDir =:= CurrentDbDir of
                            true ->
                                ok;
                            false ->
                                ?log_info("Removing all the buckets because "
                                          "database path has changed (old database path ~s)",
                                          [CurrentDbDir]),
                                [ns_orchestrator:delete_bucket(Bucket)
                                 || {Bucket, _} <- ns_bucket:get_buckets()],

                                ?log_info("Removing all unused database files"),
                                case delete_unused_buckets_db_files() of
                                    ok ->
                                        ok;
                                    Error ->
                                        Msg = iolist_to_binary(
                                                io_lib:format("Could not delete unused database files: ~p",
                                                              [Error])),
                                        {errors, [Msg]}
                                end
                        end,

                    case RV of
                        ok ->
                            cb_config_couch_sync:set_db_and_ix_paths(NewDbDir, NewIxDir);
                        _ ->
                            RV
                    end;
                error ->
                    {errors, [<<"Could not set the storage path. It must be a directory writable by 'couchbase' user.">>]}
            end;
        _ ->
            ok
    end.

%% this is RPCed by pre-2.0 nodes
local_bucket_disk_usage(BucketName) ->
    {ok, Stats} = couch_stats_reader:fetch_stats(BucketName),
    Docs = proplists:get_value(couch_docs_actual_disk_size, Stats, 0),
    Views = proplists:get_value(couch_views_actual_disk_size, Stats, 0),
    Docs + Views.

% Returns a proplist of lists of proplists.
%
% A quotaMb of none means no quota. Disks can get full, disappear,
% etc, so non-ok state is used to signal issues.
%
% NOTE: in current implementation node disk quota is supported and
% state is always ok
%
% NOTE: current node supports only single storage path and does not
% support dedicated ssd (versus hdd) path
%
% NOTE: 1.7/1.8 nodes will not have storage conf returned in current
% implementation.
%
% [{ssd, []},
%  {hdd, [[{path, /some/nice/disk/path}, {quotaMb, 1234}, {state, ok}],
%         [{path", /another/good/disk/path}, {quotaMb, 5678}, {state, ok}]]}]
%
storage_conf(Node) ->
    NodeStatus = ns_doctor:get_node(Node),
    storage_conf_from_node_status(NodeStatus).

storage_conf_from_node_status(NodeStatus) ->
    StorageConf = proplists:get_value(node_storage_conf, NodeStatus, []),
    HDDInfo = case proplists:get_value(db_path, StorageConf) of
                  undefined -> [];
                  DBDir ->
                      [{path, DBDir},
                       {index_path, proplists:get_value(index_path, StorageConf, DBDir)},
                       {quotaMb, none},
                       {state, ok}]
              end,
    [{ssd, []},
     {hdd, [HDDInfo]}].

% Quota is an integer or atom none.
% Kind is atom ssd or hdd.
%
add_storage(_Node, "", _Kind, _Quota) ->
    {error, invalid_path};

add_storage(_Node, _Path, _Kind, _Quota) ->
    % TODO.
    ok.

remove_storage(_Node, _Path) ->
    % TODO.
    {error, todo}.

node_storage_info(Node) ->
    case dict:find(Node, ns_doctor:get_nodes()) of
        {ok, NodeInfo} ->
            extract_node_storage_info(NodeInfo, Node);
        _ -> []
    end.

extract_node_storage_info(NodeInfo, Node) ->
    extract_node_storage_info(NodeInfo, Node, ns_config:get()).

extract_node_storage_info(NodeInfo, Node, Config) ->
    {RAMTotal, RAMUsed, _} = proplists:get_value(memory_data, NodeInfo),
    DiskStats = proplists:get_value(disk_data, NodeInfo),
    StorageConf = proplists:get_value(node_storage_conf, NodeInfo, []),
    DiskPaths = [X || {PropName, X} <- StorageConf,
                      PropName =:= db_path orelse PropName =:= index_path],
    case memory_quota(Node, Config) of
        MemQuotaMB when is_integer(MemQuotaMB) ->
            {DiskTotal, DiskUsed} = extract_disk_totals(DiskPaths, DiskStats),
            [{ram, [{total, RAMTotal},
                    {quotaTotal, MemQuotaMB * 1048576},
                    {used, RAMUsed},
                    {free, 0} % not used
                   ]},
             {hdd, [{total, DiskTotal},
                    {quotaTotal, DiskTotal},
                    {used, DiskUsed},
                    {free, DiskTotal - DiskUsed}
                   ]}];
        _ -> []
    end.

-spec extract_disk_totals(list(), list()) -> {integer(), integer()}.
extract_disk_totals(DiskPaths, DiskStats) ->

    F = fun (Path, {UsedMounts, ATotal, AUsed} = Tuple) ->
                case extract_disk_stats_for_path(DiskStats, Path) of
                    none -> Tuple;
                    {ok, {MPoint, KBytesTotal, Cap}} ->
                        case lists:member(MPoint, UsedMounts) of
                            true -> Tuple;
                            false ->
                                Total = KBytesTotal * 1024,
                                Used = (Total * Cap) div 100,
                                {[MPoint | UsedMounts], ATotal + Total,
                                 AUsed + Used}
                        end
                end
        end,
    {_UsedMounts, DiskTotal, DiskUsed} = lists:foldl(F, {[], 0, 0}, DiskPaths),
    {DiskTotal, DiskUsed}.

%% returns cluster_storage_info for subset of nodes
nodes_storage_info(NodeNames) ->
    NodesDict = ns_doctor:get_nodes(),
    NodesInfos = lists:foldl(fun (N, A) ->
                                     case dict:find(N, NodesDict) of
                                         {ok, V} -> [{N, V} | A];
                                         _ -> A
                                     end
                             end, [], NodeNames),
    do_cluster_storage_info(NodesInfos).

%% returns cluster storage info. This is aggregation of various
%% storage related metrics across active nodes of cluster.
%%
%% total - total amount of this resource (ram or hdd) in bytes
%%
%% free - amount of this resource free (ram or hdd) in bytes
%%
%% used - amount of this resource used (for any purpose (us, OS, other
%% processes)) in bytes
%%
%% quotaTotal - amount of quota for this resource across cluster
%% nodes. Note hdd quota is not really supported.
%%
%% quotaUsed - amount of quota already allocated
%%
%% usedByData - amount of this resource used by our data
cluster_storage_info() ->
    nodes_storage_info(ns_cluster_membership:active_nodes()).

extract_subprop(NodeInfos, Key, SubKey) ->
    [proplists:get_value(SubKey, proplists:get_value(Key, NodeInfo, [])) ||
     NodeInfo <- NodeInfos].

do_cluster_storage_info([]) -> [];
do_cluster_storage_info(NodeInfos) ->
    Config = ns_config:get(),
    AllBuckets = ns_bucket:get_buckets(Config),
    AllNodes = ordsets:intersection(lists:sort(ns_node_disco:nodes_actual_proper()),
                                    lists:sort(proplists:get_keys(NodeInfos))),
    AllNodesSize = length(AllNodes),
    RAMQuotaUsed = lists:foldl(fun ({_, BucketConfig}, RAMQuota) ->
                                       ns_bucket:raw_ram_quota(BucketConfig) * AllNodesSize + RAMQuota
                               end, 0, AllBuckets),
    StorageInfos = [extract_node_storage_info(NodeInfo, Node, Config)
                    || {Node, NodeInfo} <- NodeInfos],
    HddTotals = extract_subprop(StorageInfos, hdd, total),
    HddUsed = extract_subprop(StorageInfos, hdd, used),


    BucketsRAMUsage = lists:foldl(
                        fun ({_, Info}, RAMUsage) ->
                                case lists:keyfind(interesting_stats, 1, Info) of
                                    false ->
                                        RAMUsage;
                                    {_, InterestingStats} ->
                                        case lists:keyfind(mem_used, 1, InterestingStats) of
                                            false ->
                                                RAMUsage;
                                            {_, V} ->
                                                RAMUsage + V
                                        end
                                end
                        end, 0, NodeInfos),

    BucketsDiskUsage = lists:foldl(
                         fun ({_, Info}, DiskUsage) ->
                                 case lists:keyfind(interesting_stats, 1, Info) of
                                     false ->
                                         DiskUsage;
                                     {_, InterestingStats} ->
                                         V2 = case lists:keyfind(couch_docs_actual_disk_size, 1, InterestingStats) of
                                                  false ->
                                                      DiskUsage;
                                                  {_, X} ->
                                                      DiskUsage + X
                                              end,
                                         case lists:keyfind(couch_views_actual_disk_size, 1, InterestingStats) of
                                             false ->
                                                 V2;
                                             {_, V} ->
                                                 V2 + V
                                         end
                                 end
                         end, 0, NodeInfos),

    RAMUsed = erlang:max(lists:sum(extract_subprop(StorageInfos, ram, used)),
                         BucketsRAMUsage),
    HDDUsed = erlang:max(lists:sum(HddUsed),
                         BucketsDiskUsage),

    [{ram, [{total, lists:sum(extract_subprop(StorageInfos, ram, total))},
            {quotaTotal, lists:sum(extract_subprop(StorageInfos, ram,
                                                   quotaTotal))},
            {quotaUsed, RAMQuotaUsed},
            {used, RAMUsed},
            {usedByData, BucketsRAMUsage}
           ]},
     {hdd, [{total, lists:sum(HddTotals)},
            {quotaTotal, lists:sum(HddTotals)},
            {used, HDDUsed},
            {usedByData, BucketsDiskUsage},
            {free, lists:min(lists:zipwith(fun (A, B) -> A - B end,
                                           HddTotals, HddUsed)) * length(HddUsed)} % Minimum amount free on any node * number of nodes
           ]}].

extract_disk_stats_for_path_rec([], _Path) ->
    none;
extract_disk_stats_for_path_rec([{MountPoint0, _, _} = Info | Rest], Path) ->
    MountPoint = filename:join([MountPoint0]),  % normalize path. See filename:join docs
    MPath = case lists:reverse(MountPoint) of
                %% ends of '/'
                "/" ++ _ -> MountPoint;
                %% doesn't. Append it
                X -> lists:reverse("/" ++ X)
            end,
    case MPath =:= string:substr(Path, 1, length(MPath)) of
        true -> {ok, Info};
        _ -> extract_disk_stats_for_path_rec(Rest, Path)
    end.

%% Path0 must be an absolute path with all symlinks resolved.
extract_disk_stats_for_path(StatsList, Path0) ->
    Path = case filename:join([Path0]) of
               "/" -> "/";
               X -> X ++ "/"
           end,
    %% we sort by decreasing length so that first match is 'deepest'
    LessEqFn = fun (A,B) ->
                       length(element(1, A)) >= length(element(1, B))
               end,
    SortedList = lists:sort(LessEqFn, StatsList),
    extract_disk_stats_for_path_rec(SortedList, Path).

bucket_databases(Bucket) when is_list(Bucket) ->
    bucket_databases(list_to_binary(Bucket));
bucket_databases(Bucket) when is_binary(Bucket) ->
    couch_server:all_known_databases_with_prefix(iolist_to_binary([Bucket, $/])).

delete_couch_database(DB) ->
    RV = couch_server:delete(DB, []),
    ?log_info("Deleting database ~p: ~p~n", [DB, RV]),
    RV.

%% scan data directory for bucket names
bucket_names_from_disk() ->
    {ok, DbDir} = ns_storage_conf:this_node_dbdir(),
    Files = filelib:wildcard("*", DbDir),
    lists:foldl(
      fun (MaybeBucket, Acc) ->
              Path = filename:join(DbDir, MaybeBucket),
              case filelib:is_dir(Path) andalso
                  ns_bucket:is_valid_bucket_name(MaybeBucket) of
                  true ->
                      [MaybeBucket | Acc];
                  false ->
                      Acc
              end
      end, [], Files).

delete_disk_buckets_databases(Pred) ->
    Buckets = lists:filter(Pred, bucket_names_from_disk()),
    delete_disk_buckets_databases_loop(Pred, Buckets).

delete_disk_buckets_databases_loop(_Pred, []) ->
    ok;
delete_disk_buckets_databases_loop(Pred, [Bucket | Rest]) ->
    case delete_databases_and_files(Bucket) of
        ok ->
            delete_disk_buckets_databases_loop(Pred, Rest);
        Error ->
            Error
    end.


delete_databases_and_files(Bucket) ->
    AllDBs = bucket_databases(Bucket),
    MasterDB = iolist_to_binary([Bucket, <<"/master">>]),
    {MaybeMasterDb, RestDBs} = lists:partition(
                            fun (Name) ->
                                    Name =:= MasterDB
                            end, AllDBs),
    RV = case delete_databases_loop(MaybeMasterDb ++ RestDBs) of
             ok ->
                 ?log_info("Couch dbs are deleted. Proceeding with bucket directory"),
                 {ok, DbDir} = ns_storage_conf:this_node_dbdir(),
                 Path = filename:join(DbDir, Bucket),
                 case misc:rm_rf(Path) of
                     ok -> ok;
                     Error ->
                         ale:error(?USER_LOGGER, "Unable to rm -rf bucket database directory ~s~n~p", [Bucket, Error]),
                         Error
                 end;
             Error ->
                 ale:error(?USER_LOGGER, "Unable to delete some DBs for bucket ~s. Leaving bucket directory undeleted~n~p", [Bucket, Error]),
                 Error
         end,
    do_delete_bucket_indexes(Bucket),
    RV.

do_delete_bucket_indexes(Bucket) ->
    {ok, BaseIxDir} = this_node_ixdir(),
    IxDir = filename:join([BaseIxDir, "@indexes", Bucket]),
    case file:read_file_info(IxDir) of
        {error, enoent} ->
            ?log_debug("indexes directory doesn't exist already. fine.");
        _ ->
            ?log_debug("Going to delete indexes directory: ~p", [IxDir]),
            case misc:rm_rf(IxDir) of
                ok ->
                    ?log_debug("done deleting indexes directory");
                Error ->
                    ?log_debug("failed to delete indexes directory (~p). Will try to ignore", [Error])
            end
    end.

delete_databases_loop([]) ->
    ok;
delete_databases_loop([Db | Rest]) ->
    case delete_couch_database(Db) of
        ok ->
            delete_databases_loop(Rest);
        Error ->
            Error
    end.


%% deletes all databases files for buckets not defined for this node
%% note: this is called remotely
%%
%% it's named a bit differently from other functions here; but this function
%% is rpc called by older nodes; so we must keep this name unchanged
delete_unused_buckets_db_files() ->
    Config = ns_config:get(),
    BucketNames = ns_bucket:node_bucket_names(node(), ns_bucket:get_buckets(Config)),
    delete_disk_buckets_databases(
      fun (Bucket) ->
              RV = not(lists:member(Bucket, BucketNames)),
              case RV of
                  true ->
                      ale:info(?USER_LOGGER, "Deleting old data files of bucket ~p", [Bucket]);
                  _ ->
                      ok
              end,
              RV
      end).

-ifdef(EUNIT).
extract_disk_stats_for_path_test() ->
    DiskSupStats = [{"/",297994252,97},
             {"/lib/init/rw",1921120,1},
             {"/dev",10240,2},
             {"/dev/shm",1921120,0},
             {"/var/separate",1921120,0},
             {"/media/p2",9669472,81}],
    ?assertEqual({ok, {"/media/p2",9669472,81}},
                 extract_disk_stats_for_path(DiskSupStats,
                                             "/media/p2/mbdata")),
    ?assertEqual({ok, {"/", 297994252, 97}},
                 extract_disk_stats_for_path(DiskSupStats, "/")),
    ?assertEqual({ok, {"/", 297994252, 97}},
                 extract_disk_stats_for_path(DiskSupStats, "/lib/init")),
    ?assertEqual({ok, {"/dev", 10240, 2}},
                 extract_disk_stats_for_path(DiskSupStats, "/dev/sh")),
    ?assertEqual({ok, {"/dev", 10240, 2}},
                 extract_disk_stats_for_path(DiskSupStats, "/dev")).


-endif.

this_node_memory_data() ->
    case os:getenv("MEMBASE_RAM_MEGS") of
        false -> memsup:get_memory_data();
        X ->
            RAMBytes = list_to_integer(X) * 1048576,
            {RAMBytes, 0, 0}
    end.

allowed_node_quota_range() ->
    MemoryData = this_node_memory_data(),
    allowed_node_quota_range(MemoryData, 1024).

allowed_node_quota_range(MemoryData) ->
    allowed_node_quota_range(MemoryData, 1024).

%% when validating memory size versus cluster quota we use less strict
%% rules so that clusters upgraded from 1.6.0 are able to join
%% homogeneous nodes. See MB-2762
allowed_node_quota_range_for_joined_nodes() ->
    MemoryData = this_node_memory_data(),
    allowed_node_quota_range(MemoryData, 512).

allowed_node_quota_range(MemSupData, MinusMegs) ->
    {MaxMemoryBytes0, _, _} = MemSupData,
    MiB = 1048576,
    MinMemoryMB = 256,
    MaxMemoryMBPercent = (MaxMemoryBytes0 * 4) div (5 * MiB),
    MaxMemoryMB = lists:max([(MaxMemoryBytes0 div MiB) - MinusMegs,
                             MaxMemoryMBPercent]),
    QuotaErrorDetailsFun = fun () ->
                                   case MaxMemoryMB of
                                       MaxMemoryMBPercent ->
                                           io_lib:format(" Quota must be between 256 MB and ~w MB (80% of memory size).", [MaxMemoryMB]);
                                       _ ->
                                           io_lib:format(" Quota must be between 256 MB and ~w MB (memory size minus ~w MB).", [MaxMemoryMB, MinusMegs])
                                   end
                           end,
    {MinMemoryMB, MaxMemoryMB, QuotaErrorDetailsFun}.

default_memory_quota(MemSupData) ->
    {Min, Max, _} = allowed_node_quota_range(MemSupData),
    {MaxMemoryBytes0, _, _} = MemSupData,
    MiB = 1048576,
    Value = (MaxMemoryBytes0 * 3) div (5 * MiB),
    if Value > Max ->
            Max;
       Value < Min ->
            Min;
       true ->
            Value
    end.

{'fraction_non_alphanumeric': 0.11159935243781931, 'fraction_numerical': 0.010175977125076214, 'mean_word_length': 1.7690516388193516, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 313, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 170, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6476492', 'n_tokens_mistral': 14232, 'n_tokens_neox': 13513, 'n_words': 3128}