<commit_before>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%      http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
% A module for retrieving & configuring per-server storage paths,
% storage quotas, mem quotas, etc.
%
-module(ns_storage_conf).

-include("ns_common.hrl").

-export([memory_quota/1, change_memory_quota/2, prepare_setup_disk_storage_conf/2,
         storage_conf/1, add_storage/4, remove_storage/2,
         local_bucket_disk_usage/1, bucket_disk_usage/2]).

-export([node_storage_info/1, cluster_storage_info/0]).

-export([extract_disk_stats_for_path/2, disk_stats_for_path/2]).

memory_quota(_Node) ->
    {value, RV} = ns_config:search(ns_config:get(), memory_quota),
    RV.

change_memory_quota(_Node, NewMemQuotaMB) when is_integer(NewMemQuotaMB) ->
    ns_config:set(memory_quota, NewMemQuotaMB),
    ns_bucket:update_bucket_props("default", [{ram_quota, NewMemQuotaMB * 1048576}]).

prepare_setup_disk_storage_conf(Node, Path) when Node =:= node() ->
    {value, PropList} = ns_config:search_node(Node, ns_config:get(), memcached),
    DBDir = filename:absname(proplists:get_value(dbdir, PropList)),
    {ok, NodeInfo} = dict:find(Node, ns_doctor:get_nodes()),
    NewDBDir = filename:absname(Path),
    PathOK = case DBDir of
                 NewDBDir -> ok;
                 _ ->
                     %% TODO: check permission to write
                     filelib:ensure_dir(Path),
                     case file:make_dir(Path) of
                         ok -> ok;
                         {error, eexist} -> ok;
                         _ -> error
                     end
             end,
    case PathOK of
        ok ->
            {ok, fun () ->
                         %% we need to setup disk quota for node & default bucket
                         DiskStats = proplists:get_value(disk_data, NodeInfo),
                         %% TODO: this will not work if they put it
                         %% directly in a mount point
                         {ok, {_MPoint, KBytesTotal, Cap}} =
                             extract_disk_stats_for_path(DiskStats, NewDBDir),
                         Total = KBytesTotal * 1024,
                         Used = (Total * Cap) div 100,
                         FreeMB = (Total - Used) div 1048576,

                         ns_config:set({node, node(), memcached},
                                       [{dbdir, Path},
                                        {hdd_quota, FreeMB}
                                        | lists:keydelete(dbdir, 1, PropList)]),

                         ns_bucket:update_bucket_props("default", [{hdd_quota, FreeMB * 1048576}]),

                         ns_memcached:sync_bucket_config("default")
                 end};
        X -> X
    end.

local_bucket_disk_usage("default" = _Bucket) ->
    {value, PropList} = ns_config:search_node(node(), ns_config:get(), memcached),
    DBDir = proplists:get_value(dbdir, PropList),
    {ok, Filenames} = file:list_dir(DBDir),
    %% this doesn't include filesystem slack
    lists:sum([filelib:file_size(filename:join([DBDir, Name])) || Name <- Filenames]);
local_bucket_disk_usage(_) ->
    %% TODO
    local_bucket_disk_usage("default").

bucket_disk_usage(Node, Bucket) ->
    case rpc:call(Node, ns_storage_conf, local_bucket_disk_usage, [Bucket]) of
        {badrpc, _} ->
            0;
        X -> X
    end.
<commit_msg>don't set hdd_quota of default bucket when configuring dbpath<commit_after>%% @author Northscale <<EMAIL>>
%% @copyright 2010 NorthScale, Inc.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%      http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
% A module for retrieving & configuring per-server storage paths,
% storage quotas, mem quotas, etc.
%
-module(ns_storage_conf).

-include("ns_common.hrl").

-export([memory_quota/1, change_memory_quota/2, prepare_setup_disk_storage_conf/2,
         storage_conf/1, add_storage/4, remove_storage/2,
         local_bucket_disk_usage/1, bucket_disk_usage/2]).

-export([node_storage_info/1, cluster_storage_info/0]).

-export([extract_disk_stats_for_path/2, disk_stats_for_path/2]).

memory_quota(_Node) ->
    {value, RV} = ns_config:search(ns_config:get(), memory_quota),
    RV.

change_memory_quota(_Node, NewMemQuotaMB) when is_integer(NewMemQuotaMB) ->
    ns_config:set(memory_quota, NewMemQuotaMB),
    ns_bucket:update_bucket_props("default", [{ram_quota, NewMemQuotaMB * 1048576}]).

prepare_setup_disk_storage_conf(Node, Path) when Node =:= node() ->
    {value, PropList} = ns_config:search_node(Node, ns_config:get(), memcached),
    DBDir = filename:absname(proplists:get_value(dbdir, PropList)),
    NewDBDir = filename:absname(Path),
    PathOK = case DBDir of
                 NewDBDir -> ok;
                 _ ->
                     %% TODO: check permission to write
                     filelib:ensure_dir(Path),
                     case file:make_dir(Path) of
                         ok -> ok;
                         {error, eexist} -> ok;
                         _ -> error
                     end
             end,
    case PathOK of
        ok ->
            {ok, fun () ->
                         ns_config:set({node, node(), memcached},
                                       [{dbdir, Path}
                                        | lists:keydelete(dbdir, 1, PropList)]),

                         ns_memcached:sync_bucket_config("default")
                 end};
        X -> X
    end.

local_bucket_disk_usage("default" = _Bucket) ->
    {value, PropList} = ns_config:search_node(node(), ns_config:get(), memcached),
    DBDir = proplists:get_value(dbdir, PropList),
    {ok, Filenames} = file:list_dir(DBDir),
    %% this doesn't include filesystem slack
    lists:sum([filelib:file_size(filename:join([DBDir, Name])) || Name <- Filenames]);
local_bucket_disk_usage(_) ->
    %% TODO
    local_bucket_disk_usage("default").

bucket_disk_usage(Node, Bucket) ->
    case rpc:call(Node, ns_storage_conf, local_bucket_disk_usage, [Bucket]) of
        {badrpc, _} ->
            0;
        X -> X
    end.

{'fraction_non_alphanumeric': 0.11240255138199859, 'fraction_numerical': 0.011197732104890149, 'mean_word_length': 2.372848948374761, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '777647', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2060, 'n_words': 512}