<commit_before>-module(podrec_files).
% Copyright 2015 <NAME>
% This file is part of podcast_recoder, a project licensed under the terms of
% the GNU Affero General Public License Version 3 (see LICENSE).

-behaviour(gen_server).
%%
%% API
-export([init_file_table/1,
         start_link/1,
         get_file/2,
         exists_cached_file/1,
         update_orig_url/3,
         update_last_requested/2]).

%% gen_server callbacks
-export([init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3]).

-record(state, {jobs=#{},
                jobs_running=0,
                queue=queue:new(),
                callback}).

-include_lib("records.hrl").

%%%===================================================================
%%% Behaviour methods
%%%===================================================================

-callback start_link() -> T :: term().

-callback start_link_storage() -> T :: term().

-callback mnesia_table_name() -> TableName :: atom().

-callback get_cached_file_path(LocalName :: binary()) -> Path :: term().

-callback try_recode(OriginalFilePath :: term(), RecodedFilePath :: term()) ->
    finished | {error, Reason :: term()}.

-callback file_fetch_user_timeout() -> Milliseconds :: integer().

-callback file_recent() -> Seconds :: integer().

-callback get_file_preconfigured_url(LocalName :: binary()) ->
    {ok, Path :: term()} | {error, Reason :: term()}.

-callback get_storage_gen_server_name() -> GenServerName :: atom().

%% @doc in KiB
-callback get_max_cache_size() -> MaxCacheSize :: integer().

-callback compressible() -> Compressibility :: boolean().

%%%===================================================================
%%% API
%%%===================================================================

init_file_table(Callback) when is_atom(Callback) ->
    mnesia:create_table(Callback:mnesia_table_name(),
                        [{record_name, file},
                         {attributes, record_info(fields, file)},
                         {disc_copies, [node()]}]),
    ok.

start_link(Callback) when is_atom(Callback) ->
    gen_server:start_link({local, Callback}, ?MODULE, [Callback], []).

get_file(LocalName, Callback) when is_binary(LocalName), is_atom(Callback) ->
    lager:debug("get_file(~p, ~p)", [LocalName, Callback]),
    FileRev = podrec_storage:get_file_rev(LocalName, Callback),
    lager:debug("FileRev=~p", [FileRev]),
    R = case podrec_storage:exists_cached_file(FileRev) of
            true ->
                case podrec_storage:is_cached_file_recent(FileRev, Callback) of
                    true ->
                        lager:info("Req:~p cache is recent", [LocalName]),
                        {ok, FileRev};
                    false ->
                        lager:info("Req:~p cache is not recent, trying to fetch ...", [LocalName]),
                        case try_fetch(LocalName, Callback) of
                            {ok, _} ->
                                lager:info("Req:~p fetch successful", [LocalName]),
                                {ok, podrec_storage:get_file_rev(LocalName, Callback)};
                            % FIXME: don't serve older version on 404, return 404 and delete local version
                            {error, Reason} ->
                                lager:info("Req:~p error on fetching, serving older version (error: ~p)",
                                           [LocalName, Reason]),
                                {ok, FileRev} % just serve older version :P
                        end
                end;
            false ->
                lager:info("Req:~p not in cache, trying to fetch ...", [LocalName]),
                case try_fetch(LocalName, Callback) of
                    {ok, NewFileRevPath} ->
                        lager:info("Req:~p fetch successful", [LocalName]),
                        {ok, podrec_storage:get_file_rev(NewFileRevPath, Callback)};
                    {error, Reason} ->
                        lager:info("Req:~p error on fetching, reason: ~p", [LocalName, Reason]),
                        {error, Reason}
                end
        end,
    case R of
        {ok, _} ->
            update_last_requested(LocalName, Callback);
        {error, _Reason} ->
            ok
    end,
    R.

%% @doc Updates a file's orig_url in the database. Creates entry if necessary.
update_orig_url(LocalName, Url, Callback) when is_binary(LocalName), is_binary(Url), is_atom(Callback) ->
    lager:debug("adding ~p to database", [Url]),
    T = fun() -> File = case mnesia:read(Callback:mnesia_table_name(), LocalName) of
                            [F] -> F;
                            [] -> #file{local_name=LocalName}
                        end,
                 mnesia:write(Callback:mnesia_table_name(), File#file{orig_url=Url}, write) end,
    {atomic, ok} = mnesia:transaction(T),
    ok.

update_last_requested(LocalName,Callback) when is_binary(LocalName), is_atom(Callback) ->
    Now = erlang:system_time(seconds),<commit_msg>fix serving files just added to cache<commit_after>-module(podrec_files).
% Copyright 2015 <NAME>
% This file is part of podcast_recoder, a project licensed under the terms of
% the GNU Affero General Public License Version 3 (see LICENSE).

-behaviour(gen_server).
%%
%% API
-export([init_file_table/1,
         start_link/1,
         get_file/2,
         exists_cached_file/1,
         update_orig_url/3,
         update_last_requested/2]).

%% gen_server callbacks
-export([init/1,
         handle_call/3,
         handle_cast/2,
         handle_info/2,
         terminate/2,
         code_change/3]).

-record(state, {jobs=#{},
                jobs_running=0,
                queue=queue:new(),
                callback}).

-include_lib("records.hrl").

%%%===================================================================
%%% Behaviour methods
%%%===================================================================

-callback start_link() -> T :: term().

-callback start_link_storage() -> T :: term().

-callback mnesia_table_name() -> TableName :: atom().

-callback get_cached_file_path(LocalName :: binary()) -> Path :: term().

-callback try_recode(OriginalFilePath :: term(), RecodedFilePath :: term()) ->
    finished | {error, Reason :: term()}.

-callback file_fetch_user_timeout() -> Milliseconds :: integer().

-callback file_recent() -> Seconds :: integer().

-callback get_file_preconfigured_url(LocalName :: binary()) ->
    {ok, Path :: term()} | {error, Reason :: term()}.

-callback get_storage_gen_server_name() -> GenServerName :: atom().

%% @doc in KiB
-callback get_max_cache_size() -> MaxCacheSize :: integer().

-callback compressible() -> Compressibility :: boolean().

%%%===================================================================
%%% API
%%%===================================================================

init_file_table(Callback) when is_atom(Callback) ->
    mnesia:create_table(Callback:mnesia_table_name(),
                        [{record_name, file},
                         {attributes, record_info(fields, file)},
                         {disc_copies, [node()]}]),
    ok.

start_link(Callback) when is_atom(Callback) ->
    gen_server:start_link({local, Callback}, ?MODULE, [Callback], []).

get_file(LocalName, Callback) when is_binary(LocalName), is_atom(Callback) ->
    lager:debug("get_file(~p, ~p)", [LocalName, Callback]),
    FileRev = podrec_storage:get_file_rev(LocalName, Callback),
    lager:debug("FileRev=~p", [FileRev]),
    R = case podrec_storage:exists_cached_file(FileRev) of
            true ->
                case podrec_storage:is_cached_file_recent(FileRev, Callback) of
                    true ->
                        lager:info("Req:~p cache is recent", [LocalName]),
                        {ok, FileRev};
                    false ->
                        lager:info("Req:~p cache is not recent, trying to fetch ...", [LocalName]),
                        case try_fetch(LocalName, Callback) of
                            {ok, _} ->
                                lager:info("Req:~p fetch successful", [LocalName]),
                                {ok, podrec_storage:get_file_rev(LocalName, Callback)};
                            % FIXME: don't serve older version on 404, return 404 and delete local version
                            {error, Reason} ->
                                lager:info("Req:~p error on fetching, serving older version (error: ~p)",
                                           [LocalName, Reason]),
                                {ok, FileRev} % just serve older version :P
                        end
                end;
            false ->
                lager:info("Req:~p not in cache, trying to fetch ...", [LocalName]),
                case try_fetch(LocalName, Callback) of
                    {ok, NewFileRevPath} ->
                        lager:info("Req:~p fetch successful", [LocalName]),
                        {ok, podrec_storage:get_file_rev(LocalName, Callback)};
                    {error, Reason} ->
                        lager:info("Req:~p error on fetching, reason: ~p", [LocalName, Reason]),
                        {error, Reason}
                end
        end,
    case R of
        {ok, _} ->
            update_last_requested(LocalName, Callback);
        {error, _Reason} ->
            ok
    end,
    R.

%% @doc Updates a file's orig_url in the database. Creates entry if necessary.
update_orig_url(LocalName, Url, Callback) when is_binary(LocalName), is_binary(Url), is_atom(Callback) ->
    lager:debug("adding ~p to database", [Url]),
    T = fun() -> File = case mnesia:read(Callback:mnesia_table_name(), LocalName) of
                            [F] -> F;
                            [] -> #file{local_name=LocalName}
                        end,
                 mnesia:write(Callback:mnesia_table_name(), File#file{orig_url=Url}, write) end,
    {atomic, ok} = mnesia:transaction(T),
    ok.

update_last_requested(LocalName,Callback) when is_binary(LocalName), is_atom(Callback) ->
    Now = erlang:system_time(seconds),
{'fraction_non_alphanumeric': 0.17629119527791443, 'fraction_numerical': 0.004722085587801279, 'mean_word_length': 2.2140373063547267, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '312831', 'n_tokens_mistral': 2895, 'n_tokens_neox': 2760, 'n_words': 631}