<commit_before>-module(erl_ip_index_debug).

-export([
    test_random_results/1,
    test_random_results/8,
    test_range_results/1,
    test_range_results/6,
    now_diff_us/1,
    rebuild_bert/4,
    build_full_index/3,
    build_full_lists/2,
    benchmark_ips/4,
    benchmark/5
]).

-define(IP_LIMIT, (1 bsl 32)).
-define(RANDOM_THRESHOLD, 150000).

test_random_results([BertFile, BlacklistFile, OutputFile, Amount, Seed1, Seed2, Seed3]) ->
    test_random_results(BertFile, BlacklistFile, OutputFile, ?RANDOM_THRESHOLD, list_to_integer(Amount), list_to_integer(Seed1), list_to_integer(Seed2), list_to_integer(Seed3)).

test_random_results(BertFile, BlacklistFile, OutputFile, Threshold, Amount, Seed1, Seed2, Seed3) ->
    Index = build_full_index(BertFile, BlacklistFile, Threshold),
    random:seed(Seed1, Seed2, Seed3),
    Bin = run_random(Index, Amount),
    {ok, File} = file:open(OutputFile, [write, raw]),
    file:write(File, Bin),
    file:close(File).

run_random(Index, Amount) ->
    run_random(Index, Amount, <<>>).

run_random(_Index, 0, Bin) ->
    Bin;
run_random(Index, Amount, Bin) ->
    Ip = random:uniform(?IP_LIMIT) - 1,
    Results = lists:sort(erl_ip_index:lookup_subnet_nif(Index, Ip, 32)),
    NewBin = add_results(Results, Bin),
    run_random(Index, Amount-1, NewBin).

test_range_results([BertFile, BlacklistFile, OutputFile, Start, End]) ->
    test_range_results(BertFile, BlacklistFile, OutputFile, 1000000, list_to_integer(Start), list_to_integer(End)).

test_range_results(BertFile, BlacklistFile, OutputFile, Threshold, Start, End) ->
    {ok, File} = file:open(OutputFile, [write, raw]),
    Index = build_full_index(BertFile, BlacklistFile, Threshold),
    Bin = run_range(Index, Start, End),<commit_msg>Add mask testing<commit_after>-module(erl_ip_index_debug).

-export([
    test_random_results/1,
    test_random_results/8,
    test_range_results/1,
    test_range_results/6,
    now_diff_us/1,
    rebuild_bert/4,
    build_full_index/3,
    build_full_lists/2,
    benchmark_ips/4,
    benchmark/5
]).

-define(IP_LIMIT, (1 bsl 32)).
-define(RANDOM_THRESHOLD, 150000).

test_random_results([BertFile, BlacklistFile, OutputFile, Amount, Seed1, Seed2, Seed3]) ->
    test_random_results(BertFile, BlacklistFile, OutputFile, ?RANDOM_THRESHOLD, list_to_integer(Amount), list_to_integer(Seed1), list_to_integer(Seed2), list_to_integer(Seed3)).

test_random_results(BertFile, BlacklistFile, OutputFile, Threshold, Amount, Seed1, Seed2, Seed3) ->
    Index = build_full_index(BertFile, BlacklistFile, Threshold),
    random:seed(Seed1, Seed2, Seed3),
    Bin = run_random(Index, Amount),
    {ok, File} = file:open(OutputFile, [write, raw]),
    file:write(File, Bin),
    file:close(File).

run_random(Index, Amount) ->
    run_random(Index, Amount, <<>>).

run_random(_Index, 0, Bin) ->
    Bin;
run_random(Index, Amount, Bin) ->
    Ip = random:uniform(?IP_LIMIT) - 1,
    Mask = case random:uniform(2) of 1 -> 32; 2 -> 24 end,
    Results = lists:sort(erl_ip_index:lookup_subnet_nif(Index, Ip, Mask)),
    NewBin = add_results(Results, Bin),
    run_random(Index, Amount-1, NewBin).

test_range_results([BertFile, BlacklistFile, OutputFile, Start, End]) ->
    test_range_results(BertFile, BlacklistFile, OutputFile, 1000000, list_to_integer(Start), list_to_integer(End)).

test_range_results(BertFile, BlacklistFile, OutputFile, Threshold, Start, End) ->
    {ok, File} = file:open(OutputFile, [write, raw]),
    Index = build_full_index(BertFile, BlacklistFile, Threshold),
    Bin = run_range(Index, Start, End),
{'fraction_non_alphanumeric': 0.13411896745230079, 'fraction_numerical': 0.025533108866442198, 'mean_word_length': 5.157167530224525, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6929563', 'n_tokens_mistral': 1374, 'n_tokens_neox': 1299, 'n_words': 236}