<commit_before>-module(ddfs_master).
-behaviour(gen_server).

-export([start_link/0, stop/0]).
-export([get_tags/1,
         get_nodeinfo/1,
         get_read_nodes/0,
         gc_blacklist/0,
         gc_blacklist/1,
         choose_write_nodes/2,
         new_blob/3,
         tag_notify/2,
         tag_operation/2, tag_operation/3,
         update_nodes/1
        ]).
-export([init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3]).

-define(WEB_PORT, 8011).

-include("config.hrl").
-include("ddfs.hrl").
-include("ddfs_tag.hrl").

-type node_info() :: {node(), {non_neg_integer(), non_neg_integer()}}.
-record(state, {tags      = gb_trees:empty() :: gb_tree(),
                tag_cache = false            :: 'false' | gb_set(),

                nodes           = [] :: [node_info()],
                write_blacklist = [] :: [node()],
                read_blacklist  = [] :: [node()],
                gc_blacklist    = [] :: [node()]}).
-type state() :: #state{}.
-type replyto() :: {pid(), reference()}.

%% ===================================================================
%% API functions

start_link() ->
    error_logger:info_report([{"DDFS master starts"}]),
    case gen_server:start_link({local, ?MODULE}, ?MODULE, [], []) of
        {ok, Server} -> {ok, Server};
        {error, {already_started, Server}} -> {ok, Server}
    end.

stop() -> not_implemented.

-spec tag_operation(term(), tagname()) -> term().
tag_operation(Op, Tag) ->
    gen_server:call(?MODULE, {tag, Op, Tag}).
-spec tag_operation(term(), tagname(), non_neg_integer()) ->
                           term().
tag_operation(Op, Tag, Timeout) ->
    gen_server:call(?MODULE, {tag, Op, Tag}, Timeout).

-spec tag_notify(term(), tagname()) -> 'ok'.
tag_notify(Op, Tag) ->
    gen_server:cast(?MODULE, {tag_notify, Op, Tag}).

-spec get_nodeinfo('all') -> {'ok', [node_info()]}.
get_nodeinfo(all) ->
    gen_server:call(?MODULE, {get_nodeinfo, all}).

-spec get_read_nodes() -> {'ok', [node()], non_neg_integer()}.
get_read_nodes() ->
    gen_server:call(?MODULE, get_read_nodes).

-spec gc_blacklist() -> {'ok', [node()]}.
gc_blacklist() ->
    gen_server:call(?MODULE, gc_blacklist).

-spec choose_write_nodes(non_neg_integer(), [node()]) -> {'ok', [node()]}.
choose_write_nodes(K, Exclude) ->
    gen_server:call(?MODULE, {choose_write_nodes, K, Exclude}).

-spec get_tags('all') -> {[node()], [node()], [binary()]};
              ('safe') -> {'ok', [binary()]} | 'too_many_failed_nodes'.
get_tags(Mode) ->
    gen_server:call(?MODULE, {get_tags, Mode}).

-spec new_blob(string(), non_neg_integer(), [node()]) ->
                      'too_many_replicas' | {'ok', [nonempty_string()]}.
new_blob(Obj, K, Exclude) ->
    gen_server:call(?MODULE, {new_blob, Obj, K, Exclude}).

-spec update_nodes([{node(), boolean(), boolean()}]) -> 'ok'.
update_nodes(DDFSNodes) ->
    gen_server:cast(?MODULE, {update_nodes, DDFSNodes}).

-spec update_nodestats(gb_tree()) -> 'ok'.
update_nodestats(NewNodes) ->
    gen_server:cast(?MODULE, {update_nodestats, NewNodes}).

-spec update_tag_cache(gb_set()) -> 'ok'.
update_tag_cache(TagCache) ->
    gen_server:cast(?MODULE, {update_tag_cache, TagCache}).

-spec gc_blacklist([node()]) -> 'ok'.
gc_blacklist(Nodes) ->
    gen_server:cast(?MODULE, {gc_blacklist, Nodes}).

%% ===================================================================
%% gen_server callbacks

init(_Args) ->
    spawn_link(fun() -> monitor_diskspace() end),
    spawn_link(fun() -> ddfs_gc:start_gc(disco:get_setting("DDFS_DATA")) end),
    spawn_link(fun() -> refresh_tag_cache_proc() end),
    put(put_port, disco:get_setting("DDFS_PUT_PORT")),
    {ok, #state{}}.

handle_call(dbg_get_state, _, S) ->
    {reply, S, S};

handle_call({get_nodeinfo, all}, _From, #state{nodes = Nodes} = S) ->
    {reply, {ok, Nodes}, S};
<commit_msg>Add a hosted_tags api to ddfs_master.<commit_after>-module(ddfs_master).
-behaviour(gen_server).

-export([start_link/0, stop/0]).
-export([get_tags/1,
         get_nodeinfo/1,
         get_read_nodes/0,
         get_hosted_tags/1,
         gc_blacklist/0,
         gc_blacklist/1,
         choose_write_nodes/2,
         new_blob/3,
         tag_notify/2,
         tag_operation/2, tag_operation/3,
         update_nodes/1
        ]).
-export([init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3]).

-define(WEB_PORT, 8011).

-include("config.hrl").
-include("ddfs.hrl").
-include("ddfs_tag.hrl").

-type node_info() :: {node(), {non_neg_integer(), non_neg_integer()}}.
-record(state, {tags      = gb_trees:empty() :: gb_tree(),
                tag_cache = false            :: 'false' | gb_set(),

                nodes           = [] :: [node_info()],
                write_blacklist = [] :: [node()],
                read_blacklist  = [] :: [node()],
                gc_blacklist    = [] :: [node()]}).
-type state() :: #state{}.
-type replyto() :: {pid(), reference()}.

%% ===================================================================
%% API functions

start_link() ->
    error_logger:info_report([{"DDFS master starts"}]),
    case gen_server:start_link({local, ?MODULE}, ?MODULE, [], []) of
        {ok, Server} -> {ok, Server};
        {error, {already_started, Server}} -> {ok, Server}
    end.

stop() -> not_implemented.

-spec tag_operation(term(), tagname()) -> term().
tag_operation(Op, Tag) ->
    gen_server:call(?MODULE, {tag, Op, Tag}).
-spec tag_operation(term(), tagname(), non_neg_integer()) ->
                           term().
tag_operation(Op, Tag, Timeout) ->
    gen_server:call(?MODULE, {tag, Op, Tag}, Timeout).

-spec tag_notify(term(), tagname()) -> 'ok'.
tag_notify(Op, Tag) ->
    gen_server:cast(?MODULE, {tag_notify, Op, Tag}).

-spec get_nodeinfo('all') -> {'ok', [node_info()]}.
get_nodeinfo(all) ->
    gen_server:call(?MODULE, {get_nodeinfo, all}).

-spec get_read_nodes() -> {'ok', [node()], non_neg_integer()}.
get_read_nodes() ->
    gen_server:call(?MODULE, get_read_nodes).

-spec gc_blacklist() -> {'ok', [node()]}.
gc_blacklist() ->
    gen_server:call(?MODULE, gc_blacklist).

-spec gc_blacklist([node()]) -> 'ok'.
gc_blacklist(Nodes) ->
    gen_server:cast(?MODULE, {gc_blacklist, Nodes}).

-spec get_hosted_tags(host()) -> {'ok', [tagname()]} | {'error', term()}.
get_hosted_tags(Host) ->
    gen_server:call(?MODULE, {get_hosted_tags, Host}).

-spec choose_write_nodes(non_neg_integer(), [node()]) -> {'ok', [node()]}.
choose_write_nodes(K, Exclude) ->
    gen_server:call(?MODULE, {choose_write_nodes, K, Exclude}).

-spec get_tags('all') -> {[node()], [node()], [binary()]};
              ('safe') -> {'ok', [binary()]} | 'too_many_failed_nodes'.
get_tags(Mode) ->
    gen_server:call(?MODULE, {get_tags, Mode}).

-spec new_blob(string(), non_neg_integer(), [node()]) ->
                      'too_many_replicas' | {'ok', [nonempty_string()]}.
new_blob(Obj, K, Exclude) ->
    gen_server:call(?MODULE, {new_blob, Obj, K, Exclude}).

-spec update_nodes([{node(), boolean(), boolean()}]) -> 'ok'.
update_nodes(DDFSNodes) ->
    gen_server:cast(?MODULE, {update_nodes, DDFSNodes}).

-spec update_nodestats(gb_tree()) -> 'ok'.
update_nodestats(NewNodes) ->
    gen_server:cast(?MODULE, {update_nodestats, NewNodes}).

-spec update_tag_cache(gb_set()) -> 'ok'.
update_tag_cache(TagCache) ->
    gen_server:cast(?MODULE, {update_tag_cache, TagCache}).

%% ===================================================================
%% gen_server callbacks

init(_Args) ->
    spawn_link(fun() -> monitor_diskspace() end),
    spawn_link(fun() -> ddfs_gc:start_gc(disco:get_setting("DDFS_DATA")) end),
    spawn_link(fun() -> refresh_tag_cache_proc() end),
    put(put_port, disco:get_setting("DDFS_PUT_PORT")),
    {ok, #state{}}.

handle_call(dbg_get_state, _, S) ->
    {reply, S, S};

handle_call({get_nodeinfo, all}, _From, #state{nodes = Nodes} = S) ->
    {reply, {ok, Nodes}, S};

{'fraction_non_alphanumeric': 0.22778957890790955, 'fraction_numerical': 0.005872797700862176, 'mean_word_length': 3.494104435710275, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2952480', 'n_tokens_mistral': 2880, 'n_tokens_neox': 2806, 'n_words': 482}