<commit_before>%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et

-module(rebar_prv_update).

-behaviour(provider).

-export([init/1,
         do/1,
         format_error/1]).

-export([hex_to_index/1]).

-ifdef(TEST).
-export([cmp_/6, cmpl_/6, valid_vsn/1]).
-endif.

-include("rebar.hrl").
-include_lib("providers/include/providers.hrl").

-define(PROVIDER, update).
-define(DEPS, []).

%% ===================================================================
%% Public API
%% ===================================================================

-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
init(State) ->
    State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER},
                                                               {module, ?MODULE},
                                                               {bare, true},
                                                               {deps, ?DEPS},
                                                               {example, "rebar3 update"},
                                                               {short_desc, "Update package index."},
                                                               {desc, "Update package index."},
                                                               {opts, []}])),
    {ok, State1}.

-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
    try
        case rebar_packages:registry_dir(State) of
            {ok, RegistryDir} ->
                filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
                HexFile = filename:join(RegistryDir, "registry"),
                ?INFO("Updating package registry...", []),
                TmpDir = ec_file:insecure_mkdtemp(),
                TmpFile = filename:join(TmpDir, "packages.gz"),

                CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
                case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of
                    {ok, Url} ->
                        ?DEBUG("Fetching registry from ~p", [Url]),
                        case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]},
                                           [], [{stream, TmpFile}, {sync, true}],
                                           rebar) of
                            {ok, saved_to_file} ->
                                {ok, Data} = file:read_file(TmpFile),
                                Unzipped = zlib:gunzip(Data),
                                ok = file:write_file(HexFile, Unzipped),
                                ?INFO("Writing registry to ~s", [HexFile]),
                                hex_to_index(State),
                                {ok, State};
                            _ ->
                                ?PRV_ERROR(package_index_download)
                        end;
                    _ ->
                        ?PRV_ERROR({package_parse_cdn, CDN})
                end;
            {uri_parse_error, CDN} ->
                ?PRV_ERROR({package_parse_cdn, CDN})
        end
    catch
        _E:C ->
            ?DEBUG("Error creating package index: ~p ~p", [C, erlang:get_stacktrace()]),
            throw(?PRV_ERROR(package_index_write))
    end.

-spec format_error(any()) -> iolist().
format_error({package_parse_cdn, Uri}) ->
    io_lib:format("Failed to parse CDN url: ~p", [Uri]);
format_error(package_index_download) ->
    "Failed to download package index.";
format_error(package_index_write) ->
    "Failed to write package index.".

is_supported(<<"make">>) -> true;
is_supported(<<"rebar">>) -> true;
is_supported(<<"rebar3">>) -> true;
is_supported(_) -> false.

hex_to_index(State) ->
    {ok, RegistryDir} = rebar_packages:registry_dir(State),
    HexFile = filename:join(RegistryDir, "registry"),
    try ets:file2tab(HexFile) of
        {ok, Registry} ->
            try
                PackageIndex = filename:join(RegistryDir, "packages.idx"),
                ?INFO("Generating package index...", []),
                (catch ets:delete(?PACKAGE_TABLE)),
                ets:new(?PACKAGE_TABLE, [named_table, public]),
                ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) ->
                                  case lists:any(fun is_supported/1, BuildTools) of
                                      true ->
                                          DepsList = update_deps_list(Pkg, PkgVsn, Deps, Registry, State),
                                          HashedDeps = update_deps_hashes(DepsList),
                                          ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, HashedDeps, Checksum});
                                      false ->
                                          true
                                  end;
                             (_, _) ->
                                  true
                          end, true, Registry),

                ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) ->
                                  true;
                             ({Pkg, [Vsns=[_Vsn | _Rest]]}, _) when is_binary(Pkg) ->
                                  %% Verify the package is of the right build tool by checking if the first
                                  %% version exists in the table from the foldl above
                                  case [V || V <- Vsns, ets:member(?PACKAGE_TABLE, {Pkg, V})] of
                                      [] ->
                                          true;
                                      Vsns1 ->
                                          ets:insert(?PACKAGE_TABLE, {Pkg, Vsns1})
                                  end;
                             (_, _) ->
                                  true
                          end, true, Registry),
                ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}),
                ?INFO("Writing index to ~s", [PackageIndex]),
                ets:tab2file(?PACKAGE_TABLE, PackageIndex),
                true
            after
                catch ets:delete(Registry)
            end;
        {error, Reason} ->
            ?DEBUG("Error loading package registry: ~p", [Reason]),
            false
    catch
        _:_ ->
            fail
    end.

update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) ->
    lists:foldl(fun([Dep, DepVsn, false, AppName | _], DepsListAcc) ->
                        Dep1 = {Pkg, PkgVsn, Dep, AppName},
                        case {valid_vsn(DepVsn), DepVsn} of<commit_msg>Fix printing path with unicode names in prv_update<commit_after>%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et

-module(rebar_prv_update).

-behaviour(provider).

-export([init/1,
         do/1,
         format_error/1]).

-export([hex_to_index/1]).

-ifdef(TEST).
-export([cmp_/6, cmpl_/6, valid_vsn/1]).
-endif.

-include("rebar.hrl").
-include_lib("providers/include/providers.hrl").

-define(PROVIDER, update).
-define(DEPS, []).

%% ===================================================================
%% Public API
%% ===================================================================

-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
init(State) ->
    State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER},
                                                               {module, ?MODULE},
                                                               {bare, true},
                                                               {deps, ?DEPS},
                                                               {example, "rebar3 update"},
                                                               {short_desc, "Update package index."},
                                                               {desc, "Update package index."},
                                                               {opts, []}])),
    {ok, State1}.

-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
    try
        case rebar_packages:registry_dir(State) of
            {ok, RegistryDir} ->
                filelib:ensure_dir(filename:join(RegistryDir, "dummy")),
                HexFile = filename:join(RegistryDir, "registry"),
                ?INFO("Updating package registry...", []),
                TmpDir = ec_file:insecure_mkdtemp(),
                TmpFile = filename:join(TmpDir, "packages.gz"),

                CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN),
                case rebar_utils:url_append_path(CDN, ?REMOTE_REGISTRY_FILE) of
                    {ok, Url} ->
                        ?DEBUG("Fetching registry from ~p", [Url]),
                        case httpc:request(get, {Url, [{"User-Agent", rebar_utils:user_agent()}]},
                                           [], [{stream, TmpFile}, {sync, true}],
                                           rebar) of
                            {ok, saved_to_file} ->
                                {ok, Data} = file:read_file(TmpFile),
                                Unzipped = zlib:gunzip(Data),
                                ok = file:write_file(HexFile, Unzipped),
                                ?INFO("Writing registry to ~ts", [HexFile]),
                                hex_to_index(State),
                                {ok, State};
                            _ ->
                                ?PRV_ERROR(package_index_download)
                        end;
                    _ ->
                        ?PRV_ERROR({package_parse_cdn, CDN})
                end;
            {uri_parse_error, CDN} ->
                ?PRV_ERROR({package_parse_cdn, CDN})
        end
    catch
        _E:C ->
            ?DEBUG("Error creating package index: ~p ~p", [C, erlang:get_stacktrace()]),
            throw(?PRV_ERROR(package_index_write))
    end.

-spec format_error(any()) -> iolist().
format_error({package_parse_cdn, Uri}) ->
    io_lib:format("Failed to parse CDN url: ~p", [Uri]);
format_error(package_index_download) ->
    "Failed to download package index.";
format_error(package_index_write) ->
    "Failed to write package index.".

is_supported(<<"make">>) -> true;
is_supported(<<"rebar">>) -> true;
is_supported(<<"rebar3">>) -> true;
is_supported(_) -> false.

hex_to_index(State) ->
    {ok, RegistryDir} = rebar_packages:registry_dir(State),
    HexFile = filename:join(RegistryDir, "registry"),
    try ets:file2tab(HexFile) of
        {ok, Registry} ->
            try
                PackageIndex = filename:join(RegistryDir, "packages.idx"),
                ?INFO("Generating package index...", []),
                (catch ets:delete(?PACKAGE_TABLE)),
                ets:new(?PACKAGE_TABLE, [named_table, public]),
                ets:foldl(fun({{Pkg, PkgVsn}, [Deps, Checksum, BuildTools | _]}, _) when is_list(BuildTools) ->
                                  case lists:any(fun is_supported/1, BuildTools) of
                                      true ->
                                          DepsList = update_deps_list(Pkg, PkgVsn, Deps, Registry, State),
                                          HashedDeps = update_deps_hashes(DepsList),
                                          ets:insert(?PACKAGE_TABLE, {{Pkg, PkgVsn}, HashedDeps, Checksum});
                                      false ->
                                          true
                                  end;
                             (_, _) ->
                                  true
                          end, true, Registry),

                ets:foldl(fun({Pkg, [[]]}, _) when is_binary(Pkg) ->
                                  true;
                             ({Pkg, [Vsns=[_Vsn | _Rest]]}, _) when is_binary(Pkg) ->
                                  %% Verify the package is of the right build tool by checking if the first
                                  %% version exists in the table from the foldl above
                                  case [V || V <- Vsns, ets:member(?PACKAGE_TABLE, {Pkg, V})] of
                                      [] ->
                                          true;
                                      Vsns1 ->
                                          ets:insert(?PACKAGE_TABLE, {Pkg, Vsns1})
                                  end;
                             (_, _) ->
                                  true
                          end, true, Registry),
                ets:insert(?PACKAGE_TABLE, {package_index_version, ?PACKAGE_INDEX_VERSION}),
                ?INFO("Writing index to ~ts", [PackageIndex]),
                ets:tab2file(?PACKAGE_TABLE, PackageIndex),
                true
            after
                catch ets:delete(Registry)
            end;
        {error, Reason} ->
            ?DEBUG("Error loading package registry: ~p", [Reason]),
            false
    catch
        _:_ ->
            fail
    end.

update_deps_list(Pkg, PkgVsn, Deps, HexRegistry, State) ->
    lists:foldl(fun([Dep, DepVsn, false, AppName | _], DepsListAcc) ->
                        Dep1 = {Pkg, PkgVsn, Dep, AppName},
                        case {valid_vsn(DepVsn), DepVsn} of
{'fraction_non_alphanumeric': 0.1381294964028777, 'fraction_numerical': 0.003029155622870125, 'mean_word_length': 1.2482124616956078, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 98, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3186604', 'n_tokens_mistral': 3567, 'n_tokens_neox': 3378, 'n_words': 670}