<commit_before>%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 1996-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
-module(shell_docs).

-include_lib("kernel/include/eep48.hrl").

-export([render/2, render/3, render/4]).
-export([render_type/2, render_type/3, render_type/4]).
-export([render_callback/2, render_callback/3, render_callback/4]).

%% Used by chunks.escript in erl_docgen
-export([validate/1, normalize/1]).

%% Convinience functions
-export([get_doc/1, get_doc/3, get_type_doc/3, get_callback_doc/3]).

-record(config, { docs,
                  io_opts = io:getopts(),
                  io_columns = element(2,io:columns())
                }).

-define(ALL_ELEMENTS,[a,p,'div',br,h1,h2,h3,i,em,pre,code,ul,ol,li,dl,dt,dd]).
%% inline elements are:
-define(INLINE,[i,em,code,a]).
-define(IS_INLINE(ELEM),(((ELEM) =:= a) orelse ((ELEM) =:= code)
                         orelse ((ELEM) =:= i) orelse ((ELEM) =:= em))).
%% non-inline elements are:
-define(BLOCK,[p,'div',pre,br,ul,ol,li,dl,dt,dd,h1,h2,h3]).
-define(IS_BLOCK(ELEM),not ?IS_INLINE(ELEM)).
-define(IS_PRE(ELEM),(((ELEM) =:= pre))).

-type chunk_element_type() :: a | p | 'div' | i | br | em | pre | code | ul |
                              ol | li | dl | dt | dd | h1 | h2 | h3.
-type chunk_element_attr() :: {atom(),unicode:chardata()}.
-type chunk_element_attrs() :: [chunk_element_attr()].
-type chunk_element() :: {chunk_element_type(),chunk_element_attrs(),
                          chunk_elements()} | binary().
-type chunk_elements() :: [chunk_element()].
-type docs_v1() :: #docs_v1{}.


-spec validate(Module) -> ok when
      Module :: module() | docs_v1().
%% Simple validation of erlang doc chunk. Check that all tags are supported and
%% that the signature is correct.
validate(Module) when is_atom(Module) ->
    {ok, Doc} = code:get_doc(Module),
    validate(Doc);
validate(#docs_v1{ module_doc = MDocs, docs = AllDocs }) ->

    %% Check some macro in-variants
    AE = lists:sort(?ALL_ELEMENTS),
    AE = lists:sort(?INLINE ++ ?BLOCK),
    true = lists:all(fun(Elem) -> ?IS_INLINE(Elem) end, ?INLINE),
    true = lists:all(fun(Elem) -> ?IS_BLOCK(Elem) end, ?BLOCK),

    _ = maps:map(fun(_Key,MDoc) -> validate(MDoc,[]) end, MDocs),
    lists:foreach(fun({_,_Anno, Sig, Docs, _Meta}) ->
                      case lists:all(fun erlang:is_binary/1, Sig) of
                          false -> throw({invalid_signature,Sig});
                          true -> ok
                      end,
                      maps:map(fun(_Key,Doc) -> validate(Doc,[]) end, Docs)
              end, AllDocs),
    ok.
validate([H|T],Path) when is_tuple(H) ->
    _ = validate(H,Path),
    validate(T,Path);
validate({br,Attr,Content} = Br,Path) ->
    if Attr =:= [],
       Content =:= [] ->
            ok;
       true ->
            throw({content_to_allowed_in_br,Br,Path})
    end;
validate({Tag,Attr,Content},Path) ->
    case Tag =:= p andalso lists:member(p, Path) of
        true ->
            throw({nested_p_not_allowed,Tag,Path});
        false ->
            ok
    end,
    %% Test that there are no block tags within a pre, h1, h2 or h3
    case lists:member(pre,Path) or lists:member(h1,Path) or
        lists:member(h2,Path) or lists:member(h3,Path) of
        true when ?IS_BLOCK(Tag) ->
            throw({cannot_put_block_tag_within_pre,Tag,Path});
        _ ->
            ok
    end,
    %% Test that a block tag is not within an inline tag
    case lists:member(Tag,?BLOCK) of
        true ->
            case lists:any(fun(P) -> ?IS_INLINE(P) end, Path) of
                true ->
                    throw({cannot_put_inline_tag_outside_block, Tag, Path});
                false ->
                    ok
            end;
        false ->
            ok
    end,
    case lists:member(Tag,?ALL_ELEMENTS) of
        false ->
            throw({invalid_tag,Tag,Path});
        true ->
            ok
    end,
    case lists:all(fun({Key,Val}) -> is_atom(Key) andalso is_binary(Val) end,Attr) of
        true -> ok;
        false -> throw({invalid_attribute,{Tag,Attr}})
    end,
    validate(Content,[Tag | Path]);
validate([Chars | T], Path) when is_binary(Chars) ->
    validate(T, Path);
validate([],_) ->
    ok.

%% Follows algorithm described here:
%% * https://medium.com/@patrickbrosset/when-does-white-space-matter-in-html-b90e8a7cdd33
%% which in turn follows this:
%% * https://www.w3.org/TR/css-text-3/#white-space-processing
-spec normalize(Docs) -> NormalizedDocs when
      Docs :: chunk_elements(),
      NormalizedDocs :: chunk_elements().
normalize(Docs) ->
    Trimmed = normalize_trim(Docs,true),
    normalize_space(Trimmed).

normalize_trim(Bin,true) when is_binary(Bin) ->
    %% Remove any whitespace (except \n) before or after a newline
    NoSpace = re:replace(Bin,"[^\\S\n]*\n+[^\\S\n]*","\n",[unicode,global]),
    %% Replace any tabs with space
    NoTab = re:replace(NoSpace,"\t"," ",[unicode,global]),
    %% Replace any newlines with space
    NoNewLine = re:replace(NoTab,"\\v"," ",[unicode,global]),<commit_msg>shell_docs: Allow hidden and none as doc values<commit_after>%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 1996-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
-module(shell_docs).

-include_lib("kernel/include/eep48.hrl").

-export([render/2, render/3, render/4]).
-export([render_type/2, render_type/3, render_type/4]).
-export([render_callback/2, render_callback/3, render_callback/4]).

%% Used by chunks.escript in erl_docgen
-export([validate/1, normalize/1]).

%% Convinience functions
-export([get_doc/1, get_doc/3, get_type_doc/3, get_callback_doc/3]).

-record(config, { docs,
                  io_opts = io:getopts(),
                  io_columns = element(2,io:columns())
                }).

-define(ALL_ELEMENTS,[a,p,'div',br,h1,h2,h3,i,em,pre,code,ul,ol,li,dl,dt,dd]).
%% inline elements are:
-define(INLINE,[i,em,code,a]).
-define(IS_INLINE(ELEM),(((ELEM) =:= a) orelse ((ELEM) =:= code)
                         orelse ((ELEM) =:= i) orelse ((ELEM) =:= em))).
%% non-inline elements are:
-define(BLOCK,[p,'div',pre,br,ul,ol,li,dl,dt,dd,h1,h2,h3]).
-define(IS_BLOCK(ELEM),not ?IS_INLINE(ELEM)).
-define(IS_PRE(ELEM),(((ELEM) =:= pre))).

-type chunk_element_type() :: a | p | 'div' | i | br | em | pre | code | ul |
                              ol | li | dl | dt | dd | h1 | h2 | h3.
-type chunk_element_attr() :: {atom(),unicode:chardata()}.
-type chunk_element_attrs() :: [chunk_element_attr()].
-type chunk_element() :: {chunk_element_type(),chunk_element_attrs(),
                          chunk_elements()} | binary().
-type chunk_elements() :: [chunk_element()].
-type docs_v1() :: #docs_v1{}.


-spec validate(Module) -> ok when
      Module :: module() | docs_v1().
%% Simple validation of erlang doc chunk. Check that all tags are supported and
%% that the signature is correct.
validate(Module) when is_atom(Module) ->
    {ok, Doc} = code:get_doc(Module),
    validate(Doc);
validate(#docs_v1{ module_doc = MDocs, docs = AllDocs }) ->

    %% Check some macro in-variants
    AE = lists:sort(?ALL_ELEMENTS),
    AE = lists:sort(?INLINE ++ ?BLOCK),
    true = lists:all(fun(Elem) -> ?IS_INLINE(Elem) end, ?INLINE),
    true = lists:all(fun(Elem) -> ?IS_BLOCK(Elem) end, ?BLOCK),

    _ = validate_docs(MDocs),
    lists:foreach(fun({_,_Anno, Sig, Docs, _Meta}) ->
                      case lists:all(fun erlang:is_binary/1, Sig) of
                          false -> throw({invalid_signature,Sig});
                          true -> ok
                      end,
                      [validate_docs(Doc) || Doc <- Docs]
              end, AllDocs),
    ok.

validate_docs(hidden) ->
    ok;
validate_docs(none) ->
    ok;
validate_docs(#{} = MDocs) ->
    _ = maps:map(fun(_Key,MDoc) -> validate_docs(MDoc,[]) end, MDocs),
    ok.
validate_docs([H|T],Path) when is_tuple(H) ->
    _ = validate_docs(H,Path),
    validate_docs(T,Path);
validate_docs({br,Attr,Content} = Br,Path) ->
    if Attr =:= [],
       Content =:= [] ->
            ok;
       true ->
            throw({content_to_allowed_in_br,Br,Path})
    end;
validate_docs({Tag,Attr,Content},Path) ->
    case Tag =:= p andalso lists:member(p, Path) of
        true ->
            throw({nested_p_not_allowed,Tag,Path});
        false ->
            ok
    end,
    %% Test that there are no block tags within a pre, h1, h2 or h3
    case lists:member(pre,Path) or lists:member(h1,Path) or
        lists:member(h2,Path) or lists:member(h3,Path) of
        true when ?IS_BLOCK(Tag) ->
            throw({cannot_put_block_tag_within_pre,Tag,Path});
        _ ->
            ok
    end,
    %% Test that a block tag is not within an inline tag
    case lists:member(Tag,?BLOCK) of
        true ->
            case lists:any(fun(P) -> ?IS_INLINE(P) end, Path) of
                true ->
                    throw({cannot_put_inline_tag_outside_block, Tag, Path});
                false ->
                    ok
            end;
        false ->
            ok
    end,
    case lists:member(Tag,?ALL_ELEMENTS) of
        false ->
            throw({invalid_tag,Tag,Path});
        true ->
            ok
    end,
    case lists:all(fun({Key,Val}) -> is_atom(Key) andalso is_binary(Val) end,Attr) of
        true -> ok;
        false -> throw({invalid_attribute,{Tag,Attr}})
    end,
    validate_docs(Content,[Tag | Path]);
validate_docs([Chars | T], Path) when is_binary(Chars) ->
    validate_docs(T, Path);
validate_docs([],_) ->
    ok.

%% Follows algorithm described here:
%% * https://medium.com/@patrickbrosset/when-does-white-space-matter-in-html-b90e8a7cdd33
%% which in turn follows this:
%% * https://www.w3.org/TR/css-text-3/#white-space-processing
-spec normalize(Docs) -> NormalizedDocs when
      Docs :: chunk_elements(),
      NormalizedDocs :: chunk_elements().
normalize(Docs) ->
    Trimmed = normalize_trim(Docs,true),
    normalize_space(Trimmed).

normalize_trim(Bin,true) when is_binary(Bin) ->
    %% Remove any whitespace (except \n) before or after a newline
    NoSpace = re:replace(Bin,"[^\\S\n]*\n+[^\\S\n]*","\n",[unicode,global]),
    %% Replace any tabs with space
    NoTab = re:replace(NoSpace,"\t"," ",[unicode,global]),
    %% Replace any newlines with space
    NoNewLine = re:replace(NoTab,"\\v"," ",[unicode,global]),
{'fraction_non_alphanumeric': 0.1598529283025475, 'fraction_numerical': 0.010154950538387464, 'mean_word_length': 2.8271356783919597, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 73, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6318426', 'n_tokens_mistral': 3924, 'n_tokens_neox': 3725, 'n_words': 875}