<commit_before>%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ft=erlang ts=4 sw=4 et
%% Licensed under the Apache License, Version 2.0 (the "License"); you may not
%% use this file except in compliance with the License. You may obtain a copy of
%% the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
%% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
%% License for the specific language governing permissions and limitations under
%% the License.

-module(swarm_worker).
-include("swirl.hrl").
-behaviour(gen_server).

-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-spec test() -> term().
-endif.

%% api
-export([start_link/1,
         where_is/1,
         get_swarm_options/1,
         stop/1]).

%% callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

%% records and state
-record(state, {
          swarm_id ::  ppspp_options:swarm_id(),
          options :: ppspp_options:options()
         }).
-type state() :: #state{}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% api

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc start the server with a given set of options which includes
%% the root hash.
-spec start_link(ppspp_options:options()) ->
    ignore | {error,_} | {ok,pid()}.
start_link(Swarm_Options)  ->
    Swarm_id  = ppspp_options:get_swarm_id(Swarm_Options),
    Registration = {via, gproc, {n, l, {?MODULE, Swarm_id}}},
    gen_server:start_link(Registration, ?MODULE,
                          [Swarm_id , Swarm_Options], []).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc Stops the server.

-spec stop(ppspp_options:swarm_id()) -> ok | {error, any()}.
stop(Swarm_id) ->
    case where_is(Swarm_id) of
        {error, Reason} -> {error, Reason};
        {ok, Pid} -> gen_server:cast(Pid, stop)
    end.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc Looks up the pid for a given swarm id, either as string or binary.

-spec where_is(string() | ppspp_options:swarm_id()) -> {ok, pid()} | {error,_}.
where_is(Root_Hash) when is_list(Root_Hash) ->
    Swarm_Options = ppspp_options:use_default_options(Root_Hash),
    Swarm_id = ppspp_options:get_swarm_id(Swarm_Options),
    where_is(Swarm_id);
where_is(Swarm_id) ->
    case Pid = gproc:lookup_local_name({?MODULE, Swarm_id}) of
        undefined -> {error, ppspp_swarm_worker_not_found};
        _ -> {ok, Pid}
    end.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc Looks up the swarm options for a given swarm id, either as string or binary.

-spec get_swarm_options(string() | ppspp_options:swarm_id()) ->
    {ok, ppspp_options:options() } | {error, any()}.
get_swarm_options(Swarm) ->
    case where_is(Swarm) of
        {ok, Pid} -> gen_server:call(Pid, get_swarm_options);
        Error -> Error
    end.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% callbacks

-spec init([ppspp_options:swarm_id() | ppspp_options:options()]) -> {ok,state()}.
init([Swarm_id , Swarm_Options]) ->
    process_flag(trap_exit, true),
    ?INFO("swarm: ~p started with swarm_id:~p~n and options: ~p~n",
          [self(), Swarm_id , Swarm_Options]),
    {ok, #state{swarm_id = Swarm_id , options= Swarm_Options} }.

-spec handle_cast(stop | any(), state()) -> {noreply, state()}.
handle_cast(stop, State) -> {stop, normal, State};
handle_cast(Message, State) ->
    ?WARN("~s: unexpected cast ~p~n  in state ~p~n", [?MODULE, Message, State]),
    {noreply, State}.<commit_msg>swarm: add lookups for swarm id -> option<commit_after>%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ft=erlang ts=4 sw=4 et
%% Licensed under the Apache License, Version 2.0 (the "License"); you may not
%% use this file except in compliance with the License. You may obtain a copy of
%% the License at
%%
%%   http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
%% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
%% License for the specific language governing permissions and limitations under
%% the License.

-module(swarm_worker).
-include("swirl.hrl").
-behaviour(gen_server).

-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-spec test() -> term().
-endif.

%% api
-export([start_link/1,
         where_is/1,
         get_swarm_options/1,
         stop/1]).

%% callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).

%% records and state
-record(state, {
          swarm_id ::  ppspp_options:swarm_id(),
          options :: ppspp_options:options()
         }).
-type state() :: #state{}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% api

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc start the server with a given set of options which includes
%% the root hash.
-spec start_link(ppspp_options:options()) ->
    ignore | {error,_} | {ok,pid()}.
start_link(Swarm_Options)  ->
    Swarm_id  = ppspp_options:get_swarm_id(Swarm_Options),
    Registration = {via, gproc, {n, l, {?MODULE, Swarm_id}}},
    gen_server:start_link(Registration, ?MODULE,
                          [Swarm_id , Swarm_Options], []).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc Stops the server.

-spec stop(ppspp_options:swarm_id()) -> ok | {error, any()}.
stop(Swarm_id) ->
    case where_is(Swarm_id) of
        {error, Reason} -> {error, Reason};
        {ok, Pid} -> gen_server:cast(Pid, stop)
    end.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc Looks up the pid for a given swarm id.

-spec where_is(ppspp_options:swarm_id()) -> {ok, pid()} | {error,_}.
where_is(Swarm_id) ->
    case Pid = gproc:lookup_local_name({?MODULE, Swarm_id}) of
        undefined -> {error, ppspp_swarm_worker_not_found};
        _ -> {ok, Pid}
    end.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% @doc Looks up the swarm options for a given swarm id, either as string or binary.

-spec get_swarm_options(ppspp_options:swarm_id()) ->
    {ok, ppspp_options:options() } | {error, any()}.
get_swarm_options(Swarm_id) ->
    case gproc:lookup_local_properties({?MODULE, Swarm_id}) of
        [] -> {error, ppspp_swarm_not_registered};
        [{_Pid, Swarm_Options} | _ ] -> {ok, Swarm_Options }
    end.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% callbacks

-spec init([ppspp_options:swarm_id() | ppspp_options:options()]) -> {ok,state()}.
init([Swarm_id , Swarm_Options]) ->
    process_flag(trap_exit, true),
    gproc:add_local_property({?MODULE, Swarm_id}, Swarm_Options),
    ?INFO("swarm: ~p started with swarm_id:~p~n and options: ~p~n",
          [self(), Swarm_id , Swarm_Options]),
    {ok, #state{swarm_id = Swarm_id , options= Swarm_Options} }.

-spec handle_cast(stop | any(), state()) -> {noreply, state()}.
handle_cast(stop, State) -> {stop, normal, State};
handle_cast(Message, State) ->
    ?WARN("~s: unexpected cast ~p~n  in state ~p~n", [?MODULE, Message, State]),
    {noreply, State}.
{'fraction_non_alphanumeric': 0.2601307189542484, 'fraction_numerical': 0.004705882352941176, 'mean_word_length': 4.302148302148302, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3178185', 'n_tokens_mistral': 2485, 'n_tokens_neox': 2382, 'n_words': 598}