<commit_before>-module(disco_worker).
-behaviour(gen_server).

-export([start_link_remote/3,
         start_link/1,
         init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3,
         jobhome/1,
         event/3]).

-include("disco.hrl").

-record(state, {master :: node(),
                task :: task(),
                port :: none | port(),
                worker_send :: pid(),
                error_output :: boolean(),
                buffer :: binary(),
                parser :: worker_protocol:state(),
                runtime :: worker_runtime:state(),
                throttle :: worker_throttle:state()}).
-type state() :: #state{}.

-define(JOBHOME_TIMEOUT, 5 * 60 * 1000).
-define(PID_TIMEOUT, 30 * 1000).
-define(ERROR_TIMEOUT, 10 * 1000).
-define(MESSAGE_TIMEOUT, 30 * 1000).
-define(MAX_ERROR_BUFFER_SIZE, 100 * 1024).

-spec start_link_remote(nonempty_string(), pid(), task()) -> no_return().
start_link_remote(Host, NodeMon, Task) ->
    Node = disco:slave_node(Host),
    wait_until_node_ready(NodeMon),
    spawn_link(Node, disco_worker, start_link, [{self(), node(), Task}]),
    process_flag(trap_exit, true),
    receive
        ok -> ok;
        {'EXIT', _, Reason} ->
            exit({error, Reason});
        _ ->
            exit({error, "Internal server error: invalid_reply"})
    after 60000 ->
            exit({error, "Worker did not start in 60s"})
    end,
    wait_for_exit().

-spec wait_until_node_ready(pid()) -> 'ok'.
wait_until_node_ready(NodeMon) ->
    NodeMon ! {is_ready, self()},
    receive
        node_ready -> ok
    after 30000 ->
        exit({error, "Node unavailable"})
    end.
<commit_msg>improved error message in disco_worker<commit_after>-module(disco_worker).
-behaviour(gen_server).

-export([start_link_remote/3,
         start_link/1,
         init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3,
         jobhome/1,
         event/3]).

-include("disco.hrl").

-record(state, {master :: node(),
                task :: task(),
                port :: none | port(),
                worker_send :: pid(),
                error_output :: boolean(),
                buffer :: binary(),
                parser :: worker_protocol:state(),
                runtime :: worker_runtime:state(),
                throttle :: worker_throttle:state()}).
-type state() :: #state{}.

-define(JOBHOME_TIMEOUT, 5 * 60 * 1000).
-define(PID_TIMEOUT, 30 * 1000).
-define(ERROR_TIMEOUT, 10 * 1000).
-define(MESSAGE_TIMEOUT, 30 * 1000).
-define(MAX_ERROR_BUFFER_SIZE, 100 * 1024).

-spec start_link_remote(nonempty_string(), pid(), task()) -> no_return().
start_link_remote(Host, NodeMon, Task) ->
    Node = disco:slave_node(Host),
    wait_until_node_ready(NodeMon),
    spawn_link(Node, disco_worker, start_link, [{self(), node(), Task}]),
    process_flag(trap_exit, true),
    receive
        ok -> ok;
        {'EXIT', _, Reason} ->
            exit({error, Reason});
        _ ->
            exit({error, "Internal server error: invalid_reply"})
    after 60000 ->
            exit({error, "Worker did not start in 60s at " ++ Host})
    end,
    wait_for_exit().

-spec wait_until_node_ready(pid()) -> 'ok'.
wait_until_node_ready(NodeMon) ->
    NodeMon ! {is_ready, self()},
    receive
        node_ready -> ok
    after 30000 ->
        exit({error, "Node unavailable"})
    end.

{'fraction_non_alphanumeric': 0.15044501866207294, 'fraction_numerical': 0.031007751937984496, 'mean_word_length': 2.2652296157450795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4122321', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1088, 'n_words': 226}