<commit_before>%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et

-module(rebar_prv_dialyzer).

-behaviour(provider).

-export([init/1,
         do/1,
         format_error/1]).

-include("rebar.hrl").

-define(PROVIDER, dialyzer).
-define(DEPS, [compile]).

%% ===================================================================
%% Public API
%% ===================================================================

-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
init(State) ->
    State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER},
                                                               {module, ?MODULE},
                                                               {bare, false},
                                                               {deps, ?DEPS},
                                                               {example, "rebar dialyzer"},
                                                               {short_desc, "Run the Dialyzer analyzer on the project."},
                                                               {desc, ""},
                                                               {opts, []}])),
    {ok, State1}.

-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
    ?INFO("Dialyzer starting, this may take a while...", []),
    State1 = set_plt_location(State),
    Apps = rebar_state:project_apps(State1),
    Deps = rebar_state:get(State, all_deps, []),

    try
        {ok, State2} = update_plt(State1, Apps, Deps),
        succ_typings(State2, Apps)
    catch
        throw:{dialyzer_error, Error} ->
            {error, {?MODULE, {error_processing_apps, Error, Apps}}}
    end.

-spec format_error(any()) -> iolist().
format_error({error_processing_apps, Error, _Apps}) ->
    io_lib:format("Error in dialyzing apps: ~s", [Error]);
format_error(Reason) ->
    io_lib:format("~p", [Reason]).

%% Internal functions

set_plt_location(State) ->
    BuildDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR),
    DefaultPlt = filename:join([BuildDir, ".deps.plt"]),
    Plt = rebar_state:get(State, plt, DefaultPlt),
    rebar_state:set(State, plt, Plt).

update_plt(State, Apps, Deps) ->
    ?INFO("Updating plt...", []),
    Files = get_plt_files(State, Apps, Deps),
    case read_plt(State) of
        {ok, OldFiles} ->
            check_plt(State, OldFiles, Files);
        {error, no_such_file} ->
            build_plt(State, Files)
    end.

get_plt_files(State, Apps, Deps) ->
    case rebar_state:get(State, plt_apps) of
        undefined ->
            apps_to_files(Deps);
        PltApps ->
            app_names_to_files(PltApps, Apps ++ Deps)
    end.

apps_to_files(Apps) ->
    lists:flatmap(fun app_to_files/1, Apps).

app_to_files(App) ->
    AppDetails = rebar_app_info:app_details(App),
    Modules = proplists:get_value(modules, AppDetails, []),
    EbinDir = rebar_app_info:ebin_dir(App),
    modules_to_files(Modules, EbinDir).

modules_to_files(Modules, EbinDir) ->
    Ext = code:objfile_extension(),
    Mod2File = fun(Module) -> module_to_file(Module, EbinDir, Ext) end,
    rebar_utils:filtermap(Mod2File, Modules).

module_to_file(Module, EbinDir, Ext) ->
    File = filename:join(EbinDir, atom_to_list(Module) ++ Ext),
    case filelib:is_file(File) of
        true ->
            {true, File};
        false ->
            ?CONSOLE("Unknown module ~s", [Module]),
            false
    end.

app_names_to_files(AppNames, Apps) ->
    lists:flatmap(fun(AppName) -> app_name_to_files(AppName, Apps) end,
                  AppNames).

app_name_to_files(AppName, Apps) ->
    case rebar_app_utils:find(ec_cnv:to_binary(AppName), Apps) of
        {ok, App} ->
            app_to_files(App);
        error ->
            app_name_to_files(AppName)
    end.

app_name_to_files(AppName) ->
    case code:lib_dir(AppName) of
        {error, _} ->
            ?CONSOLE("Unknown application ~s", [AppName]),
            [];
        AppDir ->
            app_dir_to_files(AppDir, AppName)
    end.

app_dir_to_files(AppDir, AppName) ->
    EbinDir = filename:join(AppDir, "ebin"),
    AppFile = filename:join(EbinDir, atom_to_list(AppName) ++ ".app"),
    case file:consult(AppFile) of
        {ok, [{application, AppName, AppDetails}]} ->
            Modules = proplists:get_value(modules, AppDetails, []),
            modules_to_files(Modules, EbinDir);
        _ ->
            Error = io_lib:format("Could not parse ~p", [AppFile]),
            throw({dialyzer_error, Error})
    end.

read_plt(State) ->
    Plt = rebar_state:get(State, plt),
    case dialyzer:plt_info(Plt) of
        {ok, Info} ->
            Files = proplists:get_value(files, Info, []),
            {ok, Files};
        {error, no_such_file} = Error ->
            Error;
        {error, read_error} ->
            Error = io_lib:format("Could not read the PLT file ~p", [Plt]),
            throw({dialyzer_error, Error})
    end.

check_plt(State, OldList, FilesList) ->
    Old = sets:from_list(OldList),
    Files = sets:from_list(FilesList),
    Remove = sets:subtract(Old, Files),
    {ok, State1} = remove_plt(State, sets:to_list(Remove)),
    Check = sets:intersection(Files, Old),
    {ok, State2} = check_plt(State1, sets:to_list(Check)),
    Add = sets:subtract(Files, Old),
    add_plt(State2, sets:to_list(Add)).

remove_plt(State, []) ->
    {ok, State};<commit_msg>Improve default dialyzer PLT by trying to guess dependencies<commit_after>%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 et

-module(rebar_prv_dialyzer).

-behaviour(provider).

-export([init/1,
         do/1,
         format_error/1]).

-include("rebar.hrl").

-define(PROVIDER, dialyzer).
-define(DEPS, [compile]).

%% ===================================================================
%% Public API
%% ===================================================================

-spec init(rebar_state:t()) -> {ok, rebar_state:t()}.
init(State) ->
    State1 = rebar_state:add_provider(State, providers:create([{name, ?PROVIDER},
                                                               {module, ?MODULE},
                                                               {bare, false},
                                                               {deps, ?DEPS},
                                                               {example, "rebar dialyzer"},
                                                               {short_desc, "Run the Dialyzer analyzer on the project."},
                                                               {desc, ""},
                                                               {opts, []}])),
    {ok, State1}.

-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
    ?INFO("Dialyzer starting, this may take a while...", []),
    State1 = set_plt_location(State),
    Apps = rebar_state:project_apps(State1),
    Deps = rebar_state:get(State, all_deps, []),

    try
        {ok, State2} = update_plt(State1, Apps, Deps),
        succ_typings(State2, Apps)
    catch
        throw:{dialyzer_error, Error} ->
            {error, {?MODULE, {error_processing_apps, Error, Apps}}}
    end.

-spec format_error(any()) -> iolist().
format_error({error_processing_apps, Error, _Apps}) ->
    io_lib:format("Error in dialyzing apps: ~s", [Error]);
format_error(Reason) ->
    io_lib:format("~p", [Reason]).

%% Internal functions

set_plt_location(State) ->
    BuildDir = rebar_state:get(State, base_dir, ?DEFAULT_BASE_DIR),
    DefaultPlt = filename:join([BuildDir, ".deps.plt"]),
    Plt = rebar_state:get(State, plt, DefaultPlt),
    rebar_state:set(State, plt, Plt).

update_plt(State, Apps, Deps) ->
    ?INFO("Updating plt...", []),
    Files = get_plt_files(State, Apps, Deps),
    case read_plt(State) of
        {ok, OldFiles} ->
            check_plt(State, OldFiles, Files);
        {error, no_such_file} ->
            build_plt(State, Files)
    end.

get_plt_files(State, Apps, Deps) ->
    case rebar_state:get(State, plt_apps) of
        undefined ->
            default_plt_files(Apps, Deps);
        PltApps ->
            app_names_to_files(PltApps, Apps ++ Deps)
    end.

default_plt_files(Apps, Deps) ->
    DepApps = lists:flatmap(fun rebar_app_info:applications/1, Apps),
    default_plt_files(default_plt_apps() ++ DepApps, Apps, Deps, [], []).

default_plt_apps() ->
    [erts,
     kernel,
     stdlib].

default_plt_files([], _, _, _, Files) ->
    Files;
default_plt_files([AppName | DepApps], Apps, Deps, PltApps, Files) ->
    case lists:member(AppName, PltApps) of
        true ->
            default_plt_files(DepApps, Apps, Deps, PltApps, Files);
        false ->
            {DepApps2, Files2} = app_name_to_info(AppName, Apps, Deps),
            DepApps3 = DepApps2 ++ DepApps,
            Files3 = Files2 ++ Files,
            default_plt_files(DepApps3, Apps, Deps, [AppName | PltApps], Files3)
    end.

app_name_to_info(AppName, Apps, Deps) ->
    case rebar_app_utils:find(ec_cnv:to_binary(AppName), Apps) of
        {ok, App} ->
            % Don't include project app files in plt
            {rebar_app_info:applications(App), []};
        error ->
            app_name_to_info(AppName, Deps)
    end.

apps_to_files(Apps) ->
    lists:flatmap(fun app_to_files/1, Apps).

app_to_files(App) ->
    AppDetails = rebar_app_info:app_details(App),
    Modules = proplists:get_value(modules, AppDetails, []),
    EbinDir = rebar_app_info:ebin_dir(App),
    modules_to_files(Modules, EbinDir).

modules_to_files(Modules, EbinDir) ->
    Ext = code:objfile_extension(),
    Mod2File = fun(Module) -> module_to_file(Module, EbinDir, Ext) end,
    rebar_utils:filtermap(Mod2File, Modules).

module_to_file(Module, EbinDir, Ext) ->
    File = filename:join(EbinDir, atom_to_list(Module) ++ Ext),
    case filelib:is_file(File) of
        true ->
            {true, File};
        false ->
            ?CONSOLE("Unknown module ~s", [Module]),
            false
    end.

app_names_to_files(AppNames, Apps) ->
    ToFiles = fun(AppName) ->
                      {_, Files} = app_name_to_info(AppName, Apps),
                      Files
              end,
    lists:flatmap(ToFiles, AppNames).

app_name_to_info(AppName, Apps) ->
    case rebar_app_utils:find(ec_cnv:to_binary(AppName), Apps) of
        {ok, App} ->
            DepApps = rebar_app_info:applications(App),
            Files = app_to_files(App),
            {DepApps, Files};
        error ->
            app_name_to_info(AppName)
    end.

app_name_to_info(AppName) ->
    case code:lib_dir(AppName) of
        {error, _} ->
            ?CONSOLE("Unknown application ~s", [AppName]),
            {[], []};
        AppDir ->
            app_dir_to_info(AppDir, AppName)
    end.

app_dir_to_info(AppDir, AppName) ->
    EbinDir = filename:join(AppDir, "ebin"),
    AppFile = filename:join(EbinDir, atom_to_list(AppName) ++ ".app"),
    case file:consult(AppFile) of
        {ok, [{application, AppName, AppDetails}]} ->
            DepApps = proplists:get_value(applications, AppDetails, []),
            IncApps = proplists:get_value(included_applications, AppDetails,
                                          []),
            Modules = proplists:get_value(modules, AppDetails, []),
            Files = modules_to_files(Modules, EbinDir),
            {IncApps ++ DepApps, Files};
        _ ->
            Error = io_lib:format("Could not parse ~p", [AppFile]),
            throw({dialyzer_error, Error})
    end.

read_plt(State) ->
    Plt = rebar_state:get(State, plt),
    case dialyzer:plt_info(Plt) of
        {ok, Info} ->
            Files = proplists:get_value(files, Info, []),
            {ok, Files};
        {error, no_such_file} = Error ->
            Error;
        {error, read_error} ->
            Error = io_lib:format("Could not read the PLT file ~p", [Plt]),
            throw({dialyzer_error, Error})
    end.

check_plt(State, OldList, FilesList) ->
    Old = sets:from_list(OldList),
    Files = sets:from_list(FilesList),
    Remove = sets:subtract(Old, Files),
    {ok, State1} = remove_plt(State, sets:to_list(Remove)),
    Check = sets:intersection(Files, Old),
    {ok, State2} = check_plt(State1, sets:to_list(Check)),
    Add = sets:subtract(Files, Old),
    add_plt(State2, sets:to_list(Add)).

remove_plt(State, []) ->
    {ok, State};
{'fraction_non_alphanumeric': 0.15917905623787976, 'fraction_numerical': 0.003959276018099547, 'mean_word_length': 2.304939919893191, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '211359', 'n_tokens_mistral': 4121, 'n_tokens_neox': 4026, 'n_words': 750}