<commit_before>#!/usr/bin/env escript
%% -*- erlang -*-
%%! -smp disable
%% This script takes the EDoc output page, and tries to bring them to
%% a more modern format suitable for a navidb site.
%% Run with `escript docsite.erl' or as `./docsite.erl'. The script
%% will call the `./rebar doc' command and generate a static site in
%% the site/ directory, which can then be shoved wherever.
-module(docsite).
-mode(compile).
-export([main/0, main/1]).
-define(DOCPATH, "doc/").
-define(SITEPATH, "site/").

main() -> main([]).

main(_) ->
    ok = filelib:ensure_dir(?SITEPATH), % output directory
    "==> navidb (doc)\n" = os:cmd("./rebar doc"), % build docs
    Overview = ?DOCPATH "overview-summary.html",
    Modules = filelib:wildcard(filename:join(?DOCPATH, "navidb*.html")),
    [Pre,Post] = base(["index.html" | Modules]),
    [write(?SITEPATH,
           case File of
               Overview -> "index.html";
               _ -> File
           end,
           [Pre,content(File),Post])
     || File <- [Overview|Modules]],
    halt(0).

content(File) ->
    filter(
     lists:reverse(
      lists:foldl(fun("<h1>"++_ = Line, init) -> {keep, [Line]};<commit_msg>Fix docsite script for skip_deps.<commit_after>#!/usr/bin/env escript
%% -*- erlang -*-
%%! -smp disable
%% This script takes the EDoc output page, and tries to bring them to
%% a more modern format suitable for a navidb site.
%% Run with `escript docsite.erl' or as `./docsite.erl'. The script
%% will call the `./rebar doc' command and generate a static site in
%% the site/ directory, which can then be shoved wherever.
-module(docsite).
-mode(compile).
-export([main/0, main/1]).
-define(DOCPATH, "doc/").
-define(SITEPATH, "site/").

main() -> main([]).

main(_) ->
    ok = filelib:ensure_dir(?SITEPATH), % output directory
    "==> navidb (doc)\n" = os:cmd("./rebar doc skip_deps=true"), % build docs
    Overview = ?DOCPATH "overview-summary.html",
    Modules = filelib:wildcard(filename:join(?DOCPATH, "navidb*.html")),
    [Pre,Post] = base(["index.html" | Modules]),
    [write(?SITEPATH,
           case File of
               Overview -> "index.html";
               _ -> File
           end,
           [Pre,content(File),Post])
     || File <- [Overview|Modules]],
    halt(0).

content(File) ->
    filter(
     lists:reverse(
      lists:foldl(fun("<h1>"++_ = Line, init) -> {keep, [Line]};
{'fraction_non_alphanumeric': 0.17891642167156657, 'fraction_numerical': 0.0033599328013439733, 'mean_word_length': 2.9633943427620633, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3454500', 'n_tokens_mistral': 817, 'n_tokens_neox': 784, 'n_words': 218}