<commit_before>%%%-----------------------------------------------------------------------------
%%% @author <NAME> <<EMAIL>>
%%% @doc
%%% Common code shared between all workers
%%% @end
%%% Created : 29 Aor 2013 by tgrk <<EMAIL>>
%%%-----------------------------------------------------------------------------
-module(vffov_common).

-export([
         verbose/3,
         priv_dir/1,
         is_url/1,
         sanitize_urls/1,
         sanitize_url/1,
         move_to_download_dir/1,
         get_downloader/0,
         open_downloader_port/1,
         close_downloader_port/1
        ]).

%%=============================================================================
%% API
%%=============================================================================
verbose(Type, Msg, Args) ->
    case application:get_env(vffov, enable_logging, false) of
        false -> io:format(Msg ++ "\n", Args);
        true  -> lager:log(Type, Msg, Args)
    end.

priv_dir(App) ->
    case code:priv_dir(App) of
        {error, bad_name} ->
            {ok, Cwd} = file:get_cwd(),
            Cwd ++ "/" ++ "priv/";
        Priv ->
            Priv ++ "/"
    end.

is_url(S) when is_list(S) ->
    string:str(S, "http://") > 0 orelse string:str("https://", S) > 0;
is_url(_S) ->
    false.

sanitize_urls(L) ->
    lists:map(fun sanitize_url/1, L).

sanitize_url(Url) ->
    [Base, Part2] = string:tokens(Url, "?"),
    [{_K,V}] = lists:flatten(
                 lists:map(
                   fun(P) ->
                           [K,V] = string:tokens(P, "="),
                           case K =:= "v" of
                               true  -> {K,V};
                               false -> []
                           end
                   end,
                   string:tokens(Part2, "&")
                  )
                ),
    Base ++ "?v=" ++ V.


move_to_download_dir(Url) ->
    io:format("debug: move url=~p~n", [Url]),
    [_,Id] = string:tokens(Url, "v="),
    [File] = lists:filter(
               fun(F) -> string:str(F, Id) > 0 end,
               filelib:wildcard("*")
              ),
    Target = filename:join(
               application:get_env(vffov, download_dir, ""),
               File
              ),
    file:rename(File, Target).<commit_msg>Allow sanitization only form YouTube video urls<commit_after>%%%-----------------------------------------------------------------------------
%%% @author <NAME> <<EMAIL>>
%%% @doc
%%% Common code shared between all workers
%%% @end
%%% Created : 29 Aor 2013 by tgrk <<EMAIL>>
%%%-----------------------------------------------------------------------------
-module(vffov_common).

-export([
         verbose/3,
         priv_dir/1,
         is_url/1,
         sanitize_urls/1,
         sanitize_url/1,
         move_to_download_dir/1,
         get_downloader/0,
         open_downloader_port/1,
         close_downloader_port/1
        ]).

%%=============================================================================
%% API
%%=============================================================================
verbose(Type, Msg, Args) ->
    case application:get_env(vffov, enable_logging, false) of
        false -> io:format(Msg ++ "\n", Args);
        true  -> lager:log(Type, Msg, Args)
    end.

priv_dir(App) ->
    case code:priv_dir(App) of
        {error, bad_name} ->
            {ok, Cwd} = file:get_cwd(),
            Cwd ++ "/" ++ "priv/";
        Priv ->
            Priv ++ "/"
    end.

is_url(S) when is_list(S) ->
    string:str(S, "http://") > 0 orelse string:str("https://", S) > 0;
is_url(_S) ->
    false.

sanitize_urls(L) ->
    lists:map(fun sanitize_url/1, L).

sanitize_url(Url) ->
    [Base, Part2] = string:tokens(Url, "?"),
    case string:str(Base, "youtube.com") > 0 of
        true ->
            %% for YT videos remove all except video id parameter
            [{_K,V}] = lists:flatten(
                         lists:map(
                           fun(P) ->
                                   [K,V] = string:tokens(P, "="),
                                   case K =:= "v" of
                                       true  -> {K,V};
                                       false -> []
                                   end
                           end,
                           string:tokens(Part2, "&")
                          )
                        ),
            Base ++ "?v=" ++ V;
        false ->
            Url
    end.

move_to_download_dir(Url) ->
    io:format("debug: move url=~p~n", [Url]),
    [_,Id] = string:tokens(Url, "v="),
    [File] = lists:filter(
               fun(F) -> string:str(F, Id) > 0 end,
               filelib:wildcard("*")
              ),
    Target = filename:join(
               application:get_env(vffov, download_dir, ""),
               File
              ),
    file:rename(File, Target).
{'fraction_non_alphanumeric': 0.26668043792604834, 'fraction_numerical': 0.008882462301177443, 'mean_word_length': 1.347067377605429, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 52, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5732357', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1411, 'n_words': 271}