<commit_before>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2014 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% This module stores utility functions for use in other modules.
%%% @end
%%%-------------------------------------------------------------------
-module(utils).

%% API
-export([ensure_running/1, pmap/2, pforeach/2,
    record_type/1, access_token_hash/1, trim_spaces/1, ceil/1,
    aggregate_over_first_element/1, average/1, random_shuffle/1, get_values/2,
    random_element/1, get_host/1, get_host_as_atom/1, cmd/1, ensure_defined/3,
    process_info/1, process_info/2]).
-export([duration/1, adjust_duration/2]).
-export([mkdtemp/0, mkdtemp/3, rmtempdir/1]).
-export([to_binary/1]).
-export([save_file_on_hosts/3, save_file/2]).

-type time_unit() :: us | ms | s | min | h.

-define(TIMEOUT, 60000).

%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc
%% Ensures that Application is started. See {@link application}
%% @end
%%--------------------------------------------------------------------
-spec ensure_running(Application :: atom()) -> ok | {error, Reason :: term()}.
ensure_running(Application) ->
    case application:start(Application) of
        {error, {already_started, Application}} -> ok;
        EverythingElse -> EverythingElse
    end.

%%--------------------------------------------------------------------
%% @doc
%% A  parallel version of lists:map/2. See {@link lists:map/2}
%% @end
%%--------------------------------------------------------------------
-spec pmap(Fun :: fun((X :: A) -> B), L :: [A]) -> [B] | error.
pmap(Fun, L) ->
    Self = self(),
    Ref = erlang:make_ref(),
    PIDs = lists:map(fun(X) ->
        spawn(fun() -> pmap_f(Self, Ref, Fun, X) end)
    end, L),
    pmap_gather(PIDs, Ref, []).

%%--------------------------------------------------------------------
%% @doc
%% A parallel version of lists:foreach/2. See {@link lists:foreach/2}
%% @end
%%--------------------------------------------------------------------
-spec pforeach(Fun :: fun((X :: A) -> any()), L :: [A]) -> ok | error.
pforeach(Fun, L) ->
    Self = self(),
    Ref = erlang:make_ref(),
    Pids = lists:map(fun(X) ->
        spawn(fun() -> pforeach_f(Self, Ref, Fun, X) end)
    end, L),
    pforeach_gather(Pids, Ref).

%%--------------------------------------------------------------------
%% @doc
%% Returns hash of given AccessToken. Can be used to confirm user's GlobalId using GlobalRegistry.
%% @end
%%--------------------------------------------------------------------
-spec access_token_hash(AccessToken :: binary()) -> Hash :: binary().
access_token_hash(AccessToken) ->
    base64:encode(crypto:hash(sha512, AccessToken)).

%%--------------------------------------------------------------------
%% @doc
%% Gets record type for given record. Since the is now way of knowing whether
%% given tuple is record, this method behaviour is unspecified for non-record tuples.
%% @end
%%--------------------------------------------------------------------
-spec record_type(Record :: tuple()) ->
    atom() | no_return().
record_type(Record) when is_tuple(Record) ->
    element(1, Record).

%%--------------------------------------------------------------------<commit_msg>VFS-4310 Optimize pmap and pforeach<commit_after>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2014 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% This module stores utility functions for use in other modules.
%%% @end
%%%-------------------------------------------------------------------
-module(utils).

%% API
-export([ensure_running/1, pmap/2, pforeach/2,
    record_type/1, access_token_hash/1, trim_spaces/1, ceil/1,
    aggregate_over_first_element/1, average/1, random_shuffle/1, get_values/2,
    random_element/1, get_host/1, get_host_as_atom/1, cmd/1, ensure_defined/3,
    process_info/1, process_info/2]).
-export([duration/1, adjust_duration/2]).
-export([mkdtemp/0, mkdtemp/3, rmtempdir/1]).
-export([to_binary/1]).
-export([save_file_on_hosts/3, save_file/2]).

-type time_unit() :: us | ms | s | min | h.

-define(TIMEOUT, 60000).

%%%===================================================================
%%% API
%%%===================================================================

%%--------------------------------------------------------------------
%% @doc
%% Ensures that Application is started. See {@link application}
%% @end
%%--------------------------------------------------------------------
-spec ensure_running(Application :: atom()) -> ok | {error, Reason :: term()}.
ensure_running(Application) ->
    case application:start(Application) of
        {error, {already_started, Application}} -> ok;
        EverythingElse -> EverythingElse
    end.

%%--------------------------------------------------------------------
%% @doc
%% A  parallel version of lists:map/2. See {@link lists:map/2}
%% @end
%%--------------------------------------------------------------------
-spec pmap(Fun :: fun((X :: A) -> B), L :: [A]) -> [B] | error.
pmap(Fun, [X]) ->
    [catch Fun(X)];
pmap(Fun, L) ->
    Self = self(),
    Ref = erlang:make_ref(),
    PIDs = lists:map(fun(X) ->
        spawn(fun() -> pmap_f(Self, Ref, Fun, X) end)
    end, L),
    pmap_gather(PIDs, Ref, []).

%%--------------------------------------------------------------------
%% @doc
%% A parallel version of lists:foreach/2. See {@link lists:foreach/2}
%% @end
%%--------------------------------------------------------------------
-spec pforeach(Fun :: fun((X :: A) -> any()), L :: [A]) -> ok | error.
pforeach(Fun, [X]) ->
    catch Fun(X),
    ok;
pforeach(Fun, L) ->
    Self = self(),
    Ref = erlang:make_ref(),
    Pids = lists:map(fun(X) ->
        spawn(fun() -> pforeach_f(Self, Ref, Fun, X) end)
    end, L),
    pforeach_gather(Pids, Ref).

%%--------------------------------------------------------------------
%% @doc
%% Returns hash of given AccessToken. Can be used to confirm user's GlobalId using GlobalRegistry.
%% @end
%%--------------------------------------------------------------------
-spec access_token_hash(AccessToken :: binary()) -> Hash :: binary().
access_token_hash(AccessToken) ->
    base64:encode(crypto:hash(sha512, AccessToken)).

%%--------------------------------------------------------------------
%% @doc
%% Gets record type for given record. Since the is now way of knowing whether
%% given tuple is record, this method behaviour is unspecified for non-record tuples.
%% @end
%%--------------------------------------------------------------------
-spec record_type(Record :: tuple()) ->
    atom() | no_return().
record_type(Record) when is_tuple(Record) ->
    element(1, Record).

%%--------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.4279835390946502, 'fraction_numerical': 0.012894375857338821, 'mean_word_length': 1.4409106126548377, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4077643', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1868, 'n_words': 483}