<commit_before>%% See LICENSE for licensing information.
%% -*- coding: utf-8 -*-
-module(erldocs_core).

%% erldocs_core: utilities for module erldocs.

-export([ copy_static_files/1
        , build/1
        , dispatch/1

        , mapreduce/4
        , pmapreduce/4
        , pmapreduce/5

        , maybe_delete_xmerl_table/0
        ]).

-include("erldocs.hrl").

-define(log(Str, Args), io:format(Str++"\n", Args)).
-define(log(Str),       io:format(Str++"\n")).

%% API

%% @doc Copy static files
-spec copy_static_files (list()) -> ok.
copy_static_files (Conf) ->
    Dest = kf(dest, Conf),
    {ok, ErlDocsCSS} = erldocs_css_dtl:render([]),
    edoc_lib:write_file(ErlDocsCSS, Dest, "erldocs.css"),
    {ok, ErlDocsJS } =  erldocs_js_dtl:render([]),
    edoc_lib:write_file(ErlDocsJS,  Dest, "erldocs.js"),
    {ok, Jquery    } =   jquery_js_dtl:render([]),
    edoc_lib:write_file(Jquery,     Dest, "jquery.js"),
    ok.

%% @doc Parses arguments passed to script and calls
%% appropriate function.
-spec dispatch (list()) -> boolean().
dispatch (Conf) ->
    maybe_create_xmerl_table(),
    DidBuild = build([ {building_otp,is_building_otp(kf(apps,Conf))} | Conf]),
    ?log("Woot, finished"),
    DidBuild.

is_building_otp ([]) -> false;
is_building_otp ([AppDir|Rest]) ->
    case find_erlang_module(AppDir) of
        false -> is_building_otp(Rest);
        Path -> {true, Path}
    end.

find_erlang_module (AppDir) ->
    ExitFast = fun (Fn, _Acc) -> throw({found, Fn}) end,
    try filelib:fold_files(AppDir, "^erlang\\.erl$", true, ExitFast, not_found) of
        not_found -> false
    catch
        {found,Fn} ->
            ?log("building_otp: true"),
            Fn
    end.


%% @doc Build everything
-spec build (list()) -> boolean().
build (Conf) ->
    mkdir_p(kf(dest, Conf)),
    AppDirs = [Path || Path <- kf(apps,Conf), filelib:is_dir(Path)],
    IncludePaths = lists:flatmap(fun includes/1, AppDirs),

    Fun = fun (AppDir, Acc) -> build_apps(Conf, IncludePaths, bname(AppDir), AppDir, Acc) end,
    Index = lists:foldl(Fun, [], AppDirs),

    case length(kf(apps,Conf)) == length(Index) of
        true  ->
            %% Only the "[application]" (inserted by build_apps/5) are present
            %% in Index, thus:
            ?log("No documentation was generated!"),
            false;
        false ->
            ok = module_index(Conf, Index),
            ok = javascript_index(Conf, Index),
            ok = copy_static_files(Conf),
            true
    end.

build_apps (Conf, IncludePaths, AppName, AppDir, Index) ->
    ?log("Building ~s", [AppName]),
    Files = ensure_docsrc(Conf, IncludePaths, AppName, AppDir),
    Map = fun (File) -> build_file_map(Conf, AppName, bname(File,".xml"), File) end,
    [ ["app", AppName, AppName, "[application]"]
      | pmapreduce(Map, fun lists:append/2, [], Files)
      ++ Index
    ].

build_file_map (Conf, AppName, Module, File) ->
    case is_ignored(AppName, Module) of
        true ->
            ?log("HTML generation for ~s skipped - ~p", [Module, File]),
            [];
        false ->
            {Type, _Attr, Content} = read_xml(File),
            case is_buildable(Type) of
                false ->
                    ?log("HTML generation for ~s impossible - ~p", [Module, File]),
                    [];
                true ->
                    ?log("Generating HTML - ~s ~p", [Module, File]),
                    Xml = strip_whitespace(Content),
                    Funs = get_funs(AppName, Module, lists:keyfind(funcs, 1, Xml)),
                    TypeSpecs = read_xml_specs(Conf, Module),
                    ok = render(AppName, Module, Content, TypeSpecs, Conf),
                    Sum = lists:flatten(module_summary(Type, Xml)),
                    [ ["mod", AppName, Module, Sum] | Funs]
            end
    end.

module_summary (erlref, Xml) ->
    {_, [], Sum} = lists:keyfind(modulesummary, 1, Xml),
    unicode:characters_to_list(
      lists:filter(fun (X) -> not is_tuple(X) end, Sum));
module_summary (cref, Xml) ->
    {_, [], Sum} = lists:keyfind(libsummary, 1, Xml),
    Sum.

ensure_docsrc (Conf, IncludePaths, AppName, AppDir) ->
    BuildingOTP = kf(building_otp, Conf),
    %% List any doc/src/*.xml files that exist in the source files
    XMLFiles = filelib:wildcard(jname([AppDir, "doc", "src", "*.xml"])),
    HandWritten = [bname(File, ".xml") || File <- XMLFiles],

    ErlFiles = filelib:wildcard(jname( AppDir,        "*.erl" ))
        ++     filelib:wildcard(jname([AppDir, "src", "*.erl"]))
        ++ maybe_add_otp_preloaded(AppName, BuildingOTP),

    %% Generate any missing module XML
    SrcFiles =
        case BuildingOTP of
            false -> [File || File <- ErlFiles,
                              not lists:member(bname(File, ".erl"), HandWritten)];
            _ ->     []
        end,

    %% Output XML files to destination folder
    %% This prevents from polluting the source files
    TmpRoot = jname(kf(dest,Conf), ?ERLDOCS_SPECS_TMP),
    XMLDir  = jname(TmpRoot, AppName),
    mkdir_p(XMLDir ++ "/"),

    SpecsGenF = fun (File) -> gen_type_specs(IncludePaths, TmpRoot, File) end,
    lists:foreach(SpecsGenF, ErlFiles),

    %% Return the complete list of XML files
    F = fun () -> gen_docsrc(IncludePaths, AppName, AppDir, SrcFiles, XMLDir) end,
    XMLFiles
        ++ tmp_cd(XMLDir, F).

maybe_add_otp_preloaded ("erts", {true, ErlangErl}) ->
    ErlangErlAppDir = dname(dname(ErlangErl)),
    filelib:wildcard(jname([ErlangErlAppDir, "src", "*.erl"]));
maybe_add_otp_preloaded (_AppName, _) -> [].

gen_type_specs (IncludePaths, SpecsDest, ErlFile) ->
    case erlang:system_info(otp_release) of
        [$R,$1,Digit|_] when Digit < $5 -> SpecsGenModule = specs_gen__below_R15;
        "R"++_ ->                          SpecsGenModule = specs_gen__R15_to_17;
        Vsn when Vsn < "18" ->             SpecsGenModule = specs_gen__R15_to_17;
        _Otherwise ->                      SpecsGenModule = specs_gen__18_and_above
    end,
    ?log("Generating Type Specs - ~p", [ErlFile]),
    Args = ["-o"++SpecsDest] ++ ["-I"++Inc || Inc <- IncludePaths] ++ [ErlFile],
    try SpecsGenModule:main(Args)
    catch _:_SpecsGenError ->
            ?log("Error generating type specs for ~p", [ErlFile])
    end.

%% @doc
%% Read Erlang type specs from an XML file
read_xml_specs (Conf, Module) ->
    Fn = "specs_" ++ Module ++ ".xml",
    File = jname([kf(dest,Conf), ?ERLDOCS_SPECS_TMP, Fn]),
    case filelib:is_file(File) of
        false -> [];
        true ->
            case read_xml(File) of
                {error, _, _} -> [];
                {module, _, Specs} ->
                    ?log("Read XML Specs for ~s - ~p", [Fn, File]),
                    strip_whitespace(Specs)
            end
    end.

includes (AppDir) ->
    {ok, Cwd} = file:get_cwd(),
    CwdLen = length(Cwd),
    %% Remove Cwd plus trailing slash
    Relativer = fun (Path) -> string:substr(Path, CwdLen + 2) end,
    F = fun (File, Acc) -> push_dirname(Relativer, File, Acc) end,
    lists:usort(
      add_parents_too(Cwd, Relativer,<commit_msg>Do not report edoc error twice<commit_after>%% See LICENSE for licensing information.
%% -*- coding: utf-8 -*-
-module(erldocs_core).

%% erldocs_core: utilities for module erldocs.

-export([ copy_static_files/1
        , build/1
        , dispatch/1

        , mapreduce/4
        , pmapreduce/4
        , pmapreduce/5

        , maybe_delete_xmerl_table/0
        ]).

-include("erldocs.hrl").

-define(log(Str, Args), io:format(Str++"\n", Args)).
-define(log(Str),       io:format(Str++"\n")).

%% API

%% @doc Copy static files
-spec copy_static_files (list()) -> ok.
copy_static_files (Conf) ->
    Dest = kf(dest, Conf),
    {ok, ErlDocsCSS} = erldocs_css_dtl:render([]),
    edoc_lib:write_file(ErlDocsCSS, Dest, "erldocs.css"),
    {ok, ErlDocsJS } =  erldocs_js_dtl:render([]),
    edoc_lib:write_file(ErlDocsJS,  Dest, "erldocs.js"),
    {ok, Jquery    } =   jquery_js_dtl:render([]),
    edoc_lib:write_file(Jquery,     Dest, "jquery.js"),
    ok.

%% @doc Parses arguments passed to script and calls
%% appropriate function.
-spec dispatch (list()) -> boolean().
dispatch (Conf) ->
    maybe_create_xmerl_table(),
    DidBuild = build([ {building_otp,is_building_otp(kf(apps,Conf))} | Conf]),
    ?log("Woot, finished"),
    DidBuild.

is_building_otp ([]) -> false;
is_building_otp ([AppDir|Rest]) ->
    case find_erlang_module(AppDir) of
        false -> is_building_otp(Rest);
        Path -> {true, Path}
    end.

find_erlang_module (AppDir) ->
    ExitFast = fun (Fn, _Acc) -> throw({found, Fn}) end,
    try filelib:fold_files(AppDir, "^erlang\\.erl$", true, ExitFast, not_found) of
        not_found -> false
    catch
        {found,Fn} ->
            ?log("building_otp: true"),
            Fn
    end.


%% @doc Build everything
-spec build (list()) -> boolean().
build (Conf) ->
    mkdir_p(kf(dest, Conf)),
    AppDirs = [Path || Path <- kf(apps,Conf), filelib:is_dir(Path)],
    IncludePaths = lists:flatmap(fun includes/1, AppDirs),

    Fun = fun (AppDir, Acc) -> build_apps(Conf, IncludePaths, bname(AppDir), AppDir, Acc) end,
    Index = lists:foldl(Fun, [], AppDirs),

    case length(kf(apps,Conf)) == length(Index) of
        true  ->
            %% Only the "[application]" (inserted by build_apps/5) are present
            %% in Index, thus:
            ?log("No documentation was generated!"),
            false;
        false ->
            ok = module_index(Conf, Index),
            ok = javascript_index(Conf, Index),
            ok = copy_static_files(Conf),
            true
    end.

build_apps (Conf, IncludePaths, AppName, AppDir, Index) ->
    ?log("Building ~s", [AppName]),
    Files = ensure_docsrc(Conf, IncludePaths, AppName, AppDir),
    Map = fun (File) -> build_file_map(Conf, AppName, bname(File,".xml"), File) end,
    [ ["app", AppName, AppName, "[application]"]
      | pmapreduce(Map, fun lists:append/2, [], Files)
      ++ Index
    ].

build_file_map (Conf, AppName, Module, File) ->
    case is_ignored(AppName, Module) of
        true ->
            ?log("HTML generation for ~s skipped - ~p", [Module, File]),
            [];
        false ->
            {Type, _Attr, Content} = read_xml(File),
            case is_buildable(Type) of
                false ->
                    ?log("HTML generation for ~s impossible - ~p", [Module, File]),
                    [];
                true ->
                    ?log("Generating HTML - ~s ~p", [Module, File]),
                    Xml = strip_whitespace(Content),
                    Funs = get_funs(AppName, Module, lists:keyfind(funcs, 1, Xml)),
                    TypeSpecs = read_xml_specs(Conf, Module),
                    ok = render(AppName, Module, Content, TypeSpecs, Conf),
                    Sum = lists:flatten(module_summary(Type, Xml)),
                    [ ["mod", AppName, Module, Sum] | Funs]
            end
    end.

module_summary (erlref, Xml) ->
    {_, [], Sum} = lists:keyfind(modulesummary, 1, Xml),
    unicode:characters_to_list(
      lists:filter(fun (X) -> not is_tuple(X) end, Sum));
module_summary (cref, Xml) ->
    {_, [], Sum} = lists:keyfind(libsummary, 1, Xml),
    Sum.

ensure_docsrc (Conf, IncludePaths, AppName, AppDir) ->
    BuildingOTP = kf(building_otp, Conf),
    %% List any doc/src/*.xml files that exist in the source files
    XMLFiles = filelib:wildcard(jname([AppDir, "doc", "src", "*.xml"])),
    HandWritten = [bname(File, ".xml") || File <- XMLFiles],

    ErlFiles = filelib:wildcard(jname( AppDir,        "*.erl" ))
        ++     filelib:wildcard(jname([AppDir, "src", "*.erl"]))
        ++ maybe_add_otp_preloaded(AppName, BuildingOTP),

    %% Generate any missing module XML
    SrcFiles =
        case BuildingOTP of
            false -> [File || File <- ErlFiles,
                              not lists:member(bname(File, ".erl"), HandWritten)];
            _ ->     []
        end,

    %% Output XML files to destination folder
    %% This prevents from polluting the source files
    TmpRoot = jname(kf(dest,Conf), ?ERLDOCS_SPECS_TMP),
    XMLDir  = jname(TmpRoot, AppName),
    mkdir_p(XMLDir ++ "/"),

    SpecsGenF = fun (File) -> gen_type_specs(IncludePaths, TmpRoot, File) end,
    lists:foreach(SpecsGenF, ErlFiles),

    %% Return the complete list of XML files
    F = fun () -> gen_docsrc(IncludePaths, AppName, AppDir, SrcFiles, XMLDir) end,
    XMLFiles
        ++ tmp_cd(XMLDir, F).

maybe_add_otp_preloaded ("erts", {true, ErlangErl}) ->
    ErlangErlAppDir = dname(dname(ErlangErl)),
    filelib:wildcard(jname([ErlangErlAppDir, "src", "*.erl"]));
maybe_add_otp_preloaded (_AppName, _) -> [].

gen_type_specs (IncludePaths, SpecsDest, ErlFile) ->
    case erlang:system_info(otp_release) of
        [$R,$1,Digit|_] when Digit < $5 -> SpecsGenModule = specs_gen__below_R15;
        "R"++_ ->                          SpecsGenModule = specs_gen__R15_to_17;
        Vsn when Vsn < "18" ->             SpecsGenModule = specs_gen__R15_to_17;
        _Otherwise ->                      SpecsGenModule = specs_gen__18_and_above
    end,
    ?log("Generating Type Specs - ~p", [ErlFile]),
    Args = ["-o"++SpecsDest] ++ ["-I"++Inc || Inc <- IncludePaths] ++ [ErlFile],
    try SpecsGenModule:main(Args)
    catch _:_SpecsGenError -> ok
    end.

%% @doc
%% Read Erlang type specs from an XML file
read_xml_specs (Conf, Module) ->
    Fn = "specs_" ++ Module ++ ".xml",
    File = jname([kf(dest,Conf), ?ERLDOCS_SPECS_TMP, Fn]),
    case filelib:is_file(File) of
        false -> [];
        true ->
            case read_xml(File) of
                {error, _, _} -> [];
                {module, _, Specs} ->
                    ?log("Read XML Specs for ~s - ~p", [Fn, File]),
                    strip_whitespace(Specs)
            end
    end.

includes (AppDir) ->
    {ok, Cwd} = file:get_cwd(),
    CwdLen = length(Cwd),
    %% Remove Cwd plus trailing slash
    Relativer = fun (Path) -> string:substr(Path, CwdLen + 2) end,
    F = fun (File, Acc) -> push_dirname(Relativer, File, Acc) end,
    lists:usort(
      add_parents_too(Cwd, Relativer,
{'fraction_non_alphanumeric': 0.1502616320181021, 'fraction_numerical': 0.004384104087116391, 'mean_word_length': 2.483497536945813, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7180532', 'n_tokens_mistral': 4967, 'n_tokens_neox': 4755, 'n_words': 1086}