<commit_before>-module(mzb_py).

-export([
    start_interpreter/1,
    stop_interpreter/1,
    eval/2,
    apply/4
]).

% for tests:
-export([encode_for_python/1,encode_str_for_python/1]).

-record(python_interpreter, {
    metrics_pipe                        :: port(),
    metrics_pipe_name                   :: string(),
    python_port                         :: port()
}).

-type python_interpreter() :: #python_interpreter{}.

-spec start_interpreter(string()) -> python_interpreter().
start_interpreter(Module) ->
    {MetricsPipeName, MetricsPipe} = create_fifo(Module),
    PythonExec = os:find_executable("python"),
    PythonPort = open_port({spawn, io_lib:format("~s -i", [PythonExec])}, [
        {line, 255}, 
        {env, [
            {"MZ_PYTHON_WORKER_FIFO_NAME", MetricsPipeName}
        ]},
        exit_status, 
        eof,
        stderr_to_stdout
    ]),
    Interpreter = #python_interpreter{
        metrics_pipe        = MetricsPipe,
        metrics_pipe_name   = MetricsPipeName,
        python_port         = PythonPort
    },

    % skip version, copyright, etc
    skip_port_messages(PythonPort, 3),

    port_command(PythonPort, "import sys; sys.ps1 = ''; sys.ps2 = ''\n"),   % Get rid of standard Python prompt
    port_command(PythonPort, "import os\n"),

    {ok, WorkerDirs} = application:get_env(mzbench, workers_dirs),
    WorkersDirsStr = string:join(["'./src'" | [io_lib:format("os.path.expanduser('~s')", [filename:join(W, Module)]) || W <- WorkerDirs]], ", "),
    port_command(PythonPort, io_lib:format("sys.path = sys.path + [~s]\n", [WorkersDirsStr])),

    port_command(PythonPort, "import traceback\n"),
    port_command(PythonPort, "import mzbench\n"),
    port_command(PythonPort, mzb_string:format("import ~s\n", [Module])),
    Interpreter.


-spec stop_interpreter(python_interpreter()) -> ok.
stop_interpreter(#python_interpreter{
                            metrics_pipe        = MetricsPipe, 
                            metrics_pipe_name   = MetricsPipeName, 
                            python_port         = PythonPort
                        }) ->
    _ = (catch port_command(PythonPort, "quit()\n")),
    _ = (catch wait_python_stop(PythonPort)),
    close_fifo(MetricsPipeName, MetricsPipe).


-spec apply(python_interpreter(), string() | atom(), string() | atom(), [term()]) -> term().
apply(Interpreter, Module, Func, Args) ->
    ParamStr = string:join(lists:map(fun encode_for_python/1, Args), ", "),
    send_command(Interpreter, "~s.~s(~s)", [Module, Func, ParamStr]).

-spec eval(python_interpreter(), string()) -> term().
eval(Interpreter, Str) ->
    send_command(Interpreter, Str, []).
<commit_msg>Python worker now prepends its paths before sys.path and not after<commit_after>-module(mzb_py).

-export([
    start_interpreter/1,
    stop_interpreter/1,
    eval/2,
    apply/4
]).

% for tests:
-export([encode_for_python/1,encode_str_for_python/1]).

-record(python_interpreter, {
    metrics_pipe                        :: port(),
    metrics_pipe_name                   :: string(),
    python_port                         :: port()
}).

-type python_interpreter() :: #python_interpreter{}.

-spec start_interpreter(string()) -> python_interpreter().
start_interpreter(Module) ->
    {MetricsPipeName, MetricsPipe} = create_fifo(Module),
    PythonExec = os:find_executable("python"),
    PythonPort = open_port({spawn, io_lib:format("~s -i", [PythonExec])}, [
        {line, 255}, 
        {env, [
            {"MZ_PYTHON_WORKER_FIFO_NAME", MetricsPipeName}
        ]},
        exit_status, 
        eof,
        stderr_to_stdout
    ]),
    Interpreter = #python_interpreter{
        metrics_pipe        = MetricsPipe,
        metrics_pipe_name   = MetricsPipeName,
        python_port         = PythonPort
    },

    % skip version, copyright, etc
    skip_port_messages(PythonPort, 3),

    port_command(PythonPort, "import sys; sys.ps1 = ''; sys.ps2 = ''\n"),   % Get rid of standard Python prompt
    port_command(PythonPort, "import os\n"),

    {ok, WorkerDirs} = application:get_env(mzbench, workers_dirs),
    WorkersDirsStr = string:join(["'./src'" | [io_lib:format("os.path.expanduser('~s')", [filename:join(W, Module)]) || W <- WorkerDirs]], ", "),
    port_command(PythonPort, io_lib:format("sys.path = [~s] + sys.path\n", [WorkersDirsStr])),

    port_command(PythonPort, "import traceback\n"),
    port_command(PythonPort, "import mzbench\n"),
    port_command(PythonPort, mzb_string:format("import ~s\n", [Module])),
    Interpreter.


-spec stop_interpreter(python_interpreter()) -> ok.
stop_interpreter(#python_interpreter{
                            metrics_pipe        = MetricsPipe, 
                            metrics_pipe_name   = MetricsPipeName, 
                            python_port         = PythonPort
                        }) ->
    _ = (catch port_command(PythonPort, "quit()\n")),
    _ = (catch wait_python_stop(PythonPort)),
    close_fifo(MetricsPipeName, MetricsPipe).


-spec apply(python_interpreter(), string() | atom(), string() | atom(), [term()]) -> term().
apply(Interpreter, Module, Func, Args) ->
    ParamStr = string:join(lists:map(fun encode_for_python/1, Args), ", "),
    send_command(Interpreter, "~s.~s(~s)", [Module, Func, ParamStr]).

-spec eval(python_interpreter(), string()) -> term().
eval(Interpreter, Str) ->
    send_command(Interpreter, Str, []).

{'fraction_non_alphanumeric': 0.14884151992585729, 'fraction_numerical': 0.004819277108433735, 'mean_word_length': 2.9044862518089727, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2341535', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1772, 'n_words': 296}