<commit_before>-module(disco_worker).
-behaviour(gen_server).

-export([start_link/1, start_link_remote/1, remote_worker/1]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, 
        terminate/2, code_change/3, slave_name/1]).

-record(state, {id, master, master_url, eventserv, port, from, jobname, 
                partid, mode, child_pid, node, input, linecount, errlines, 
                results, last_msg, msg_counter, oob, oob_counter}).

-define(MAX_MSG_LENGTH, 8192).
-define(RATE_WINDOW, 100000). % 100ms
-define(RATE_LIMIT, 10).
-define(OOB_MAX, 1000).
-define(OOB_KEY_MAX, 256).

-define(SLAVE_ARGS, "+K true").
-define(CMD, "nice -n 19 $DISCO_WORKER '~s' '~s' '~s' '~s' '~w' ~s").
-define(PORT_OPT, [{line, 100000}, binary, exit_status,
                   use_stdio, stderr_to_stdout, 
                   {env, [{"LD_LIBRARY_PATH", "lib"}]}]).

get_env(Var) ->
        get_env(Var, lists:flatten(io_lib:format(" -env ~s ~~s", [Var]))).

get_env(Var, Fmt) ->
        case os:getenv(Var) of
                false -> "";
                Val -> io_lib:format(Fmt, [Val])
        end.

slave_env() ->
    lists:flatten([?SLAVE_ARGS, 
                   get_env("DISCO_MASTER_HOME", " -pa ~s/ebin"),
                   [get_env(X) || X <- ["DISCO_MASTER_PORT",
                                        "DISCO_ROOT",
                                        "DISCO_PORT",
                                        "DISCO_WORKER",
                                        "DISCO_FLAGS",
                                        "PYTHONPATH"]]]).

slave_name(Node) ->
        {ok, Name} = application:get_env(disco_name), %get_env("DISCO_NAME"),
        SName = lists:flatten([Name, "_slave"]),
        list_to_atom(SName ++ "@" ++ Node).

start_link_remote([Master, EventServ, From, JobName, PartID, 
        Mode, Node, Input]) -><commit_msg>bugfix: ignore empty environment variables when setting the slave environment<commit_after>-module(disco_worker).
-behaviour(gen_server).

-export([start_link/1, start_link_remote/1, remote_worker/1]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, 
        terminate/2, code_change/3, slave_name/1]).

-record(state, {id, master, master_url, eventserv, port, from, jobname, 
                partid, mode, child_pid, node, input, linecount, errlines, 
                results, last_msg, msg_counter, oob, oob_counter}).

-define(MAX_MSG_LENGTH, 8192).
-define(RATE_WINDOW, 100000). % 100ms
-define(RATE_LIMIT, 10).
-define(OOB_MAX, 1000).
-define(OOB_KEY_MAX, 256).

-define(SLAVE_ARGS, "+K true").
-define(CMD, "nice -n 19 $DISCO_WORKER '~s' '~s' '~s' '~s' '~w' ~s").
-define(PORT_OPT, [{line, 100000}, binary, exit_status,
                   use_stdio, stderr_to_stdout, 
                   {env, [{"LD_LIBRARY_PATH", "lib"}]}]).

get_env(Var) ->
        get_env(Var, lists:flatten(io_lib:format(" -env ~s '~~s'", [Var]))).

get_env(Var, Fmt) ->
        case os:getenv(Var) of
                false -> "";
                "" -> "";
                Val -> io_lib:format(Fmt, [Val])
        end.

slave_env() ->
    lists:flatten([?SLAVE_ARGS, 
                   get_env("DISCO_MASTER_HOME", " -pa ~s/ebin"),
                   [get_env(X) || X <- ["DISCO_MASTER_PORT",
                                        "DISCO_ROOT",
                                        "DISCO_PORT",
                                        "DISCO_WORKER",
                                        "DISCO_FLAGS",
                                        "PYTHONPATH"]]]).

slave_name(Node) ->
        {ok, Name} = application:get_env(disco_name), %get_env("DISCO_NAME"),
        SName = lists:flatten([Name, "_slave"]),
        list_to_atom(SName ++ "@" ++ Node).

start_link_remote([Master, EventServ, From, JobName, PartID, 
        Mode, Node, Input]) ->
{'fraction_non_alphanumeric': 0.15930599369085174, 'fraction_numerical': 0.02103049421661409, 'mean_word_length': 2.0636070853462156, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2983587', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1259, 'n_words': 239}