<commit_before>#!/usr/bin/env perl6
use v6;

# this script isn't in bin/ because it's not meant
# to be installed.

use Pod::To::HTML;
use URI::Escape;
use lib 'lib';
use Perl6::TypeGraph;
use Perl6::TypeGraph::Viz;

sub url-munge($_) {
    return $_ if m{^ <[a..z]>+ '://'};
    return "/type/$_" if m/^<[A..Z]>/;
    return "/routine/$_" if m/^<[a..z]>/;
    return $_;
}

my $*DEBUG = False;

my $tg;
my %names;
my %types;
my %routines;
my %methods-by-type;
my %operators;
my $footer;

sub p2h($pod) {
    pod2html($pod, :url(&url-munge), :$footer);
}

sub pod-gist(Pod::Block $pod, $level = 0) {
    my $leading = ' ' x $level;
    my %confs;
    my @chunks;
    for <config name level caption type> {
        my $thing = $pod.?"$_"();
        if $thing {
            %confs{$_} = $thing ~~ Iterable ?? $thing.perl !! $thing.Str;
        }
    }
    @chunks = $leading, $pod.^name, (%confs.perl if %confs), "\n";
    for $pod.content.list -> $c {
        if $c ~~ Pod::Block {
            @chunks.push: pod-gist($c, $level + 2);
        }
        else {
            @chunks.push: $c.indent($level + 2), "\n";
        }
    }
    @chunks.join;
}

sub recursive-dir($dir) {
    my @todo = $dir;
    gather while @todo {
        my $d = @todo.shift;
        for dir($d) -> $f {
            if $f.f {
                take $f;
            }
            else {
                @todo.push($f.path);
            }
        }
    }
}

sub MAIN(Bool :$debug, Bool :$typegraph = False) {
    $*DEBUG = $debug;
    for '', <type language routine images op op/prefix op/postfix op/infix
             op/circumfix op/postcircumfix> {
        mkdir "html/$_" unless "html/$_".IO ~~ :e;
    }

    say 'Reading lib/ ...';
    my @source = recursive-dir('lib').grep(*.f).grep(rx{\.pod$});
    @source.=map: {; .path.subst('lib/', '').subst(rx{\.pod$}, '').subst(:g, '/', '::') => $_ };
    say '... done';

    say "Reading type graph ...";
    $tg = Perl6::TypeGraph.new-from-file('type-graph.txt');
    {
        my %h = $tg.sorted.kv.flat.reverse;
        @source.=sort: { %h{.key} // -1 };
    }
    say "... done";

    $footer = footer-html;



    for (@source) {
        my $podname  = .key;
        my $file     = .value;
        my $what     = $podname ~~ /^<[A..Z]> | '::'/  ?? 'type' !! 'language';
        say "$file.path() => $what/$podname";
        %names{$podname}{$what}.push: "/$what/$podname";
        %types{$what}{$podname} =    "/$what/$podname";
        my $pod  = eval slurp($file.path) ~ "\n\$=pod";
        $pod.=[0];
        if $what eq 'language' {
            spurt "html/$what/$podname.html", p2h($pod);
            if $podname eq 'operators' {
                my @chunks = chunks-grep($pod.content,<commit_msg>[htmlify] simplify code a wee bit<commit_after>#!/usr/bin/env perl6
use v6;

# this script isn't in bin/ because it's not meant
# to be installed.

use Pod::To::HTML;
use URI::Escape;
use lib 'lib';
use Perl6::TypeGraph;
use Perl6::TypeGraph::Viz;

sub url-munge($_) {
    return $_ if m{^ <[a..z]>+ '://'};
    return "/type/$_" if m/^<[A..Z]>/;
    return "/routine/$_" if m/^<[a..z]>/;
    return $_;
}

my $*DEBUG = False;

my $tg;
my %names;
my %types;
my %routines;
my %methods-by-type;
my %operators;
my $footer = footer-html;

sub p2h($pod) {
    pod2html($pod, :url(&url-munge), :$footer);
}

sub pod-gist(Pod::Block $pod, $level = 0) {
    my $leading = ' ' x $level;
    my %confs;
    my @chunks;
    for <config name level caption type> {
        my $thing = $pod.?"$_"();
        if $thing {
            %confs{$_} = $thing ~~ Iterable ?? $thing.perl !! $thing.Str;
        }
    }
    @chunks = $leading, $pod.^name, (%confs.perl if %confs), "\n";
    for $pod.content.list -> $c {
        if $c ~~ Pod::Block {
            @chunks.push: pod-gist($c, $level + 2);
        }
        else {
            @chunks.push: $c.indent($level + 2), "\n";
        }
    }
    @chunks.join;
}

sub recursive-dir($dir) {
    my @todo = $dir;
    gather while @todo {
        my $d = @todo.shift;
        for dir($d) -> $f {
            if $f.f {
                take $f;
            }
            else {
                @todo.push($f.path);
            }
        }
    }
}

sub MAIN(Bool :$debug, Bool :$typegraph = False) {
    $*DEBUG = $debug;
    for '', <type language routine images op op/prefix op/postfix op/infix
             op/circumfix op/postcircumfix> {
        mkdir "html/$_" unless "html/$_".IO ~~ :e;
    }

    say 'Reading lib/ ...';
    my @source = recursive-dir('lib').grep(*.f).grep(rx{\.pod$});
    @source.=map: {; .path.subst('lib/', '').subst(rx{\.pod$}, '').subst(:g, '/', '::') => $_ };
    say '... done';

    say "Reading type graph ...";
    $tg = Perl6::TypeGraph.new-from-file('type-graph.txt');
    {
        my %h = $tg.sorted.kv.flat.reverse;
        @source.=sort: { %h{.key} // -1 };
    }
    say "... done";


    for (@source) {
        my $podname  = .key;
        my $file     = .value;
        my $what     = $podname ~~ /^<[A..Z]> | '::'/  ?? 'type' !! 'language';
        say "$file.path() => $what/$podname";
        %names{$podname}{$what}.push: "/$what/$podname";
        %types{$what}{$podname} =    "/$what/$podname";
        my $pod  = eval slurp($file.path) ~ "\n\$=pod";
        $pod.=[0];
        if $what eq 'language' {
            spurt "html/$what/$podname.html", p2h($pod);
            if $podname eq 'operators' {
                my @chunks = chunks-grep($pod.content,
{'fraction_non_alphanumeric': 0.21571140816700238, 'fraction_numerical': 0.004761032777879509, 'mean_word_length': 1.9161772557394554, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1908869', 'n_tokens_mistral': 2134, 'n_tokens_neox': 2063, 'n_words': 461}