<commit_before>#!/usr/bin/env perl6
use v6;

# This script isn't in bin/ because it's not meant to be installed.
# For syntax highlighting, needs pygmentize version 2.0 or newer installed
#
# for doc.perl6.org, the build process goes like this:
# * a cron job on hack.p6c.org as user 'doc.perl6.org' triggers the rebuild.
# It looks like this:
#
# */5 * * * * flock -n ~/update.lock -c ./doc/util/update-and-sync > update.log 2>&1
#
# util/update-and-sync is under version control in the perl6/doc repo (same as
# this file), and it first updtes the git repository. If something changed, it
# run htmlify, captures the output, and on success, syncs both the generated
# files and the logs. In case of failure, only the logs are synchronized.
#
# The build logs are available at http://doc.perl6.org/build-log/
#

BEGIN say 'Initializing ...';

use Pod::To::HTML;
use URI::Escape;
use lib 'lib';
use Perl6::TypeGraph;
use Perl6::TypeGraph::Viz;
use Perl6::Documentable::Registry;
use Pod::Convenience;
use Pod::Htmlify;

my $*DEBUG = False;

my $type-graph;
my %routines-by-type;
my %*POD2HTML-CALLBACKS;

# TODO: Generate menulist automatically
my @menu =
    ('language',''         ) => (),
    ('type', 'Types'       ) => <basic composite domain-specific exceptions>,
    ('routine', 'Routines' ) => <sub method term operator>,
#    ('module', 'Modules'   ) => (),
#    ('formalities',''      ) => ();
;

my $head   = slurp 'template/head.html';
sub header-html ($current-selection = 'nothing selected') is cached {
    state $header = slurp 'template/header.html';

    my $menu-items = [~]
        q[<div class="menu-items dark-green">],
        @menu>>.key.map(-> ($dir, $name) {qq[
            <a class="menu-item {$dir eq $current-selection ?? "selected darker-green" !! ""}"
                href="/$dir.html">
                { $name || $dir.wordcase }
            </a>
        ]}), #"
        q[</div>];

    my $sub-menu-items = '';
    state %sub-menus = @menu>>.key>>[0] Z=> @menu>>.value;
    if %sub-menus{$current-selection} -> $_ {
        $sub-menu-items = [~]
            q[<div class="menu-items darker-green">],
            qq[<a class="menu-item" href="/$current-selection.html">All</a>],
            .map({qq[
                <a class="menu-item" href="/$current-selection\-$_.html">
                    {.wordcase}
                </a>
            ]}),
            q[</div>]
    }

    state $menu-pos = ($header ~~ /MENU/).from;
    $header.subst('MENU', :p($menu-pos), $menu-items ~ $sub-menu-items);
}

sub p2h($pod, $selection = 'nothing selected', :$pod-path = 'unknown') {
    pod2html $pod,
        :url(&url-munge),
        :$head,
        :header(header-html $selection),
        :footer(footer-html($pod-path)),
        :default-title("Perl 6 Documentation"),
}

sub recursive-dir($dir) {
    my @todo = $dir;
    gather while @todo {
        my $d = @todo.shift;
        for dir($d) -> $f {
            if $f.f {
                take $f;
            }
            else {
                @todo.push($f.path);
            }
        }
    }
}

# --sparse=5: only process 1/5th of the files
# mostly useful for performance optimizations, profiling etc.
sub MAIN(
    Bool :$debug,
    Bool :$typegraph = False,
    Int  :$sparse,
    Bool :$disambiguation = True,
    Bool :$search-file = True,
    Bool :$no-highlight = False,
    Bool :$no-inline-python = False,
) {
    $*DEBUG = $debug;

    say 'Creating html/ subdirectories ...';
    for '', <type language routine images syntax> {
        mkdir "html/$_" unless "html/$_".IO ~~ :e;
    }
<commit_msg>Fix list flattening issue in html subdir creation loop.<commit_after>#!/usr/bin/env perl6
use v6;

# This script isn't in bin/ because it's not meant to be installed.
# For syntax highlighting, needs pygmentize version 2.0 or newer installed
#
# for doc.perl6.org, the build process goes like this:
# * a cron job on hack.p6c.org as user 'doc.perl6.org' triggers the rebuild.
# It looks like this:
#
# */5 * * * * flock -n ~/update.lock -c ./doc/util/update-and-sync > update.log 2>&1
#
# util/update-and-sync is under version control in the perl6/doc repo (same as
# this file), and it first updtes the git repository. If something changed, it
# run htmlify, captures the output, and on success, syncs both the generated
# files and the logs. In case of failure, only the logs are synchronized.
#
# The build logs are available at http://doc.perl6.org/build-log/
#

BEGIN say 'Initializing ...';

use Pod::To::HTML;
use URI::Escape;
use lib 'lib';
use Perl6::TypeGraph;
use Perl6::TypeGraph::Viz;
use Perl6::Documentable::Registry;
use Pod::Convenience;
use Pod::Htmlify;

my $*DEBUG = False;

my $type-graph;
my %routines-by-type;
my %*POD2HTML-CALLBACKS;

# TODO: Generate menulist automatically
my @menu =
    ('language',''         ) => (),
    ('type', 'Types'       ) => <basic composite domain-specific exceptions>,
    ('routine', 'Routines' ) => <sub method term operator>,
#    ('module', 'Modules'   ) => (),
#    ('formalities',''      ) => ();
;

my $head   = slurp 'template/head.html';
sub header-html ($current-selection = 'nothing selected') is cached {
    state $header = slurp 'template/header.html';

    my $menu-items = [~]
        q[<div class="menu-items dark-green">],
        @menu>>.key.map(-> ($dir, $name) {qq[
            <a class="menu-item {$dir eq $current-selection ?? "selected darker-green" !! ""}"
                href="/$dir.html">
                { $name || $dir.wordcase }
            </a>
        ]}), #"
        q[</div>];

    my $sub-menu-items = '';
    state %sub-menus = @menu>>.key>>[0] Z=> @menu>>.value;
    if %sub-menus{$current-selection} -> $_ {
        $sub-menu-items = [~]
            q[<div class="menu-items darker-green">],
            qq[<a class="menu-item" href="/$current-selection.html">All</a>],
            .map({qq[
                <a class="menu-item" href="/$current-selection\-$_.html">
                    {.wordcase}
                </a>
            ]}),
            q[</div>]
    }

    state $menu-pos = ($header ~~ /MENU/).from;
    $header.subst('MENU', :p($menu-pos), $menu-items ~ $sub-menu-items);
}

sub p2h($pod, $selection = 'nothing selected', :$pod-path = 'unknown') {
    pod2html $pod,
        :url(&url-munge),
        :$head,
        :header(header-html $selection),
        :footer(footer-html($pod-path)),
        :default-title("Perl 6 Documentation"),
}

sub recursive-dir($dir) {
    my @todo = $dir;
    gather while @todo {
        my $d = @todo.shift;
        for dir($d) -> $f {
            if $f.f {
                take $f;
            }
            else {
                @todo.push($f.path);
            }
        }
    }
}

# --sparse=5: only process 1/5th of the files
# mostly useful for performance optimizations, profiling etc.
sub MAIN(
    Bool :$debug,
    Bool :$typegraph = False,
    Int  :$sparse,
    Bool :$disambiguation = True,
    Bool :$search-file = True,
    Bool :$no-highlight = False,
    Bool :$no-inline-python = False,
) {
    $*DEBUG = $debug;

    say 'Creating html/ subdirectories ...';
    for flat '', <type language routine images syntax> {
        mkdir "html/$_" unless "html/$_".IO ~~ :e;
    }

{'fraction_non_alphanumeric': 0.1701509068254188, 'fraction_numerical': 0.0063685449259310535, 'mean_word_length': 2.3899577663068983, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6902437', 'n_tokens_mistral': 2491, 'n_tokens_neox': 2379, 'n_words': 649}