<commit_before>-module(statser_finder).

-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.

-export([find_metrics/1,
         find_metrics/2]).


find_metrics(Parts) ->
    find_metrics(Parts, ["."]).


find_metrics([FilePart], Cwd) ->
    MetricsPath = tl(Cwd),
    Files0 = glob_whispers(FilePart, full_dir(Cwd)),
    Files = lists:map(fun(F) -> to_file(MetricsPath, F) end, Files0),
    Dirs0 = glob_dirs(FilePart, full_dir(Cwd)),
    Dirs = lists:map(fun(D) -> to_dir(MetricsPath, D) end, Dirs0),
    Files ++ Dirs;

find_metrics([DirPart | Rest], Cwd) ->
    DirCandidates = glob_dirs(DirPart, full_dir(Cwd)),
    Descend = fun(Dir) -> find_metrics(Rest, Cwd ++ [Dir]) end,
    lists:flatmap(Descend, DirCandidates).


to_file(Path, File) ->
    to_node(Path, File, true).


to_dir(Path, Dir) ->
    to_node(Path, Dir, false).


to_node(Path, Node, IsLeaf) ->
    {[{<<"leaf">>, IsLeaf},
      {<<"allowChildren">>, not IsLeaf},
      {<<"expandable">>, not IsLeaf},
      {<<"text">>, list_to_binary(Node)},
      {<<"id">>, to_path(Path ++ [Node])}]}.


to_path(Path) ->
    list_to_binary(string:join(Path, ".")).


glob_dirs(Dir, Cwd) when is_binary(Dir) ->
    glob_dirs(binary_to_list(Dir), Cwd);

glob_dirs(Dir, Cwd) ->
    [D || D <- filelib:wildcard(Dir, Cwd),
          filelib:is_dir(Cwd ++ "/" ++ D)].


glob_whispers(File, Cwd) when is_binary(File) ->
    glob_whispers(binary_to_list(File), Cwd);

glob_whispers(File, Cwd) ->
    WhisperFile = File ++ ".wsp",
    [drop_suffix(F) || F <- filelib:wildcard(WhisperFile, Cwd)].
<commit_msg>api: sort metrics results<commit_after>-module(statser_finder).

-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
-endif.

-export([find_metrics/1,
         find_metrics/2]).


find_metrics(Parts) ->
    find_metrics(Parts, ["."]).


find_metrics([FilePart], Cwd) ->
    MetricsPath = tl(Cwd),
    Files = glob_whispers(FilePart, full_dir(Cwd)),
    Dirs = glob_dirs(FilePart, full_dir(Cwd)),
    Elements = lists:sort(fun by_name/2, Files ++ Dirs),
    lists:map(fun(E) -> to_node(E, MetricsPath) end, Elements);

find_metrics([DirPart | Rest], Cwd) ->
    DirCandidates = glob_dirs(DirPart, full_dir(Cwd)),
    Descend = fun({dir, Dir}) -> find_metrics(Rest, Cwd ++ [Dir]) end,
    lists:flatmap(Descend, DirCandidates).


to_node({file, File}, Path) ->
    to_node(File, Path, true);

to_node({dir, Dir}, Path) ->
    to_node(Dir, Path, false).


to_node(Node, Path, IsLeaf) ->
    {[{<<"leaf">>, IsLeaf},
      {<<"allowChildren">>, not IsLeaf},
      {<<"expandable">>, not IsLeaf},
      {<<"text">>, list_to_binary(Node)},
      {<<"id">>, to_path(Path ++ [Node])}]}.


to_path(Path) ->
    list_to_binary(string:join(Path, ".")).


by_name({_TypeA, NameA}, {_TypeB, NameB}) when NameA =< NameB -> true;
by_name({_TypeA, _NameA}, {_TypeB, _NameB}) -> false.


glob_dirs(Dir, Cwd) when is_binary(Dir) ->
    glob_dirs(binary_to_list(Dir), Cwd);

glob_dirs(Dir, Cwd) ->
    [{dir, D} || D <- filelib:wildcard(Dir, Cwd),
                 filelib:is_dir(Cwd ++ "/" ++ D)].


glob_whispers(File, Cwd) when is_binary(File) ->
    glob_whispers(binary_to_list(File), Cwd);

glob_whispers(File, Cwd) ->
    WhisperFile = File ++ ".wsp",
    [{file, drop_suffix(F)} || F <- filelib:wildcard(WhisperFile, Cwd)].

{'fraction_non_alphanumeric': 0.1997563946406821, 'fraction_numerical': 0.0027405602923264312, 'mean_word_length': 3.9103139013452917, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '200697', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1307, 'n_words': 223}