<commit_before>-module(n2o_file).
-author('<NAME>').
-include_lib("n2o/include/wf.hrl").
-include_lib("kernel/include/file.hrl").
-compile(export_all).
%                     1       2      3      4       5       6      7     8      9      10       11
% -record(ftp,     { sid, filename, hash, source, target, offset, meta, data, status, block, priority }).

% temporaries
-define(ROOT, code:priv_dir(n2o)).
-define(next, 1024).
-define(stop, 0).

% N2O Protocols

info(#ftp{status= <<"init">>, offset=Size, block=B, data=Msg}=FTP, Req, State) ->
    application:set_env(n2o,formatter,bert),

    File = filename:join([?ROOT,wf:to_list(FTP#ftp.sid),FTP#ftp.filename]),
    FSize = case file:read_file_info(File) of {ok, Fi} -> Fi#file_info.size; {error, _} -> 0 end,

    wf:info(?MODULE,"File Transfer Init: ~p : size:~p~n",[File, FSize]),
    wf:info(?MODULE, "Size: ~p block:~p==0", [Size, B]),

    if FSize >= Size andalso B==0 -><commit_msg>1MB chunk<commit_after>-module(n2o_file).
-author('<NAME>').
-include_lib("n2o/include/wf.hrl").
-include_lib("kernel/include/file.hrl").
-compile(export_all).
%                     1       2      3      4       5       6      7     8      9      10       11
% -record(ftp,     { sid, filename, hash, source, target, offset, meta, data, status, block, priority }).

% temporaries
-define(ROOT, code:priv_dir(n2o)).
-define(next, 1024*1024).
-define(stop, 0).

% N2O Protocols

info(#ftp{status= <<"init">>, offset=Size, block=B, data=Msg}=FTP, Req, State) ->
    application:set_env(n2o,formatter,bert),

    File = filename:join([?ROOT,wf:to_list(FTP#ftp.sid),FTP#ftp.filename]),
    FSize = case file:read_file_info(File) of {ok, Fi} -> Fi#file_info.size; {error, _} -> 0 end,

    wf:info(?MODULE,"File Transfer Init: ~p : size:~p~n",[File, FSize]),
    wf:info(?MODULE, "Size: ~p block:~p==0", [Size, B]),

    if FSize >= Size andalso B==0 ->
{'fraction_non_alphanumeric': 0.19281563655573164, 'fraction_numerical': 0.030110935023771792, 'mean_word_length': 3.171806167400881, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4908427', 'n_tokens_mistral': 758, 'n_tokens_neox': 704, 'n_words': 149}