<commit_before>-module(temp_gc).

-include_lib("kernel/include/file.hrl").

-export([start_link/6]).

-define(GC_INTERVAL, 600000).

start_link(Master, EventServer, DdfsMaster, DataRoot, Node, GCAfter) ->
    put(master, Master),
    put(events, EventServer),
    put(ddfs, DdfsMaster),
    put(root, filename:join(DataRoot, Node)),
    put(gcafter, GCAfter),
    loop().

loop() ->
    case catch {gen_server:call(get(master), get_purged),
                gen_server:call(get(events), get_jobs)} of
        {{ok, Purged}, {ok, Jobs}} ->
            {ok, Dirs} = prim_file:list_dir(get(root)),
            Active = gb_sets:from_list(
                [Name || {Name, active, _Start, _Pid} <- Jobs]),
            process_dir(Dirs, gb_sets:from_ordset(Purged), Active);
        _ ->
            % master busy, try again after GC_INTERVAL
            ok
    end,
    timer:sleep(?GC_INTERVAL),
    loop().

process_dir([], _Purged, _Active) -> ok;
process_dir([Dir|R], Purged, Active) ->
    Path = filename:join(get(root), Dir),
    {ok, Jobs} = prim_file:list_dir(Path),
    [process_job(filename:join(Path, Job), Purged) ||
        Job <- Jobs, ifdead(Job, Active)],
    process_dir(R, Purged, Active).

ifdead(Job, Active) -><commit_msg>bugfix: try again later if the data root is missing in temp_gc<commit_after>-module(temp_gc).

-include_lib("kernel/include/file.hrl").

-export([start_link/6]).

-define(GC_INTERVAL, 600000).

start_link(Master, EventServer, DdfsMaster, DataRoot, Node, GCAfter) ->
    put(master, Master),
    put(events, EventServer),
    put(ddfs, DdfsMaster),
    put(root, filename:join(DataRoot, Node)),
    put(gcafter, GCAfter),
    loop().

loop() ->
    case catch {gen_server:call(get(master), get_purged),
                gen_server:call(get(events), get_jobs)} of
        {{ok, Purged}, {ok, Jobs}} ->
            case prim_file:list_dir(get(root)) of
                {ok, Dirs} ->
                    Active = gb_sets:from_list(
                        [Name || {Name, active, _Start, _Pid} <- Jobs]),
                    process_dir(Dirs, gb_sets:from_ordset(Purged), Active);
                _ ->
                    % fresh install, try again after GC_INTERVAL
                    ok
            end;
        _ ->
            % master busy, try again after GC_INTERVAL
            ok
    end,
    timer:sleep(?GC_INTERVAL),
    loop().

process_dir([], _Purged, _Active) -> ok;
process_dir([Dir|R], Purged, Active) ->
    Path = filename:join(get(root), Dir),
    {ok, Jobs} = prim_file:list_dir(Path),
    [process_job(filename:join(Path, Job), Purged) ||
        Job <- Jobs, ifdead(Job, Active)],
    process_dir(R, Purged, Active).

ifdead(Job, Active) ->
{'fraction_non_alphanumeric': 0.15276232851316277, 'fraction_numerical': 0.005190952910641453, 'mean_word_length': 2.641025641025641, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6681481', 'n_tokens_mistral': 952, 'n_tokens_neox': 907, 'n_words': 189}