<commit_before>unit module Pod::Htmlify;

use URI::Escape;

#| Escape special characters in URLs if necessary
sub url-munge($_) is export {
    return $_ if m{^ <[a..z]>+ '://'};
    return "/type/{uri_escape $_}" if m/^<[A..Z]>/;
    return "/routine/{uri_escape $_}" if m/^<[a..z]>|^<-alpha>*$/;
    # poor man's <identifier>
    if m/ ^ '&'( \w <[[\w'-]>* ) $/ {
        return "/routine/{uri_escape $0}";
    }
    return $_;
}

constant badchars = qw[$ / \ . % ? & = # + " ' : ~ < >];
my \goodnames = badchars.map: '$' ~ *.uniname.subst(' ', '_', :g);
constant length = badchars.elems;

use experimental :cached;

sub escape-filename($s is copy) is export is cached {
#    return $s if $s ~~ m{^ <[a..z]>+ '://'}; # bail on external links

    loop (my int $i = 0;$i < length;$i++) {
        $s = $s.subst(badchars[$i], goodnames[$i], :g)
    }

    $s
}

sub unescape-percent($s) is cached {
    $s.subst(:g, / [ '%' (<.xdigit> ** 2 ) ]+ /, -> $/ { Buf.new($0.flatmap({ :16(~$_) })).decode('UTF-8') })
}

sub rewrite-url($s) is export is cached {
    my Str $r;
    given $s {
        when / ^ [ 'http' | 'https' | 'irc' ] '://' / {
            # external link, we bail<commit_msg>| doesn't work on ntfs either, added<commit_after>unit module Pod::Htmlify;

use URI::Escape;

#| Escape special characters in URLs if necessary
sub url-munge($_) is export {
    return $_ if m{^ <[a..z]>+ '://'};
    return "/type/{uri_escape $_}" if m/^<[A..Z]>/;
    return "/routine/{uri_escape $_}" if m/^<[a..z]>|^<-alpha>*$/;
    # poor man's <identifier>
    if m/ ^ '&'( \w <[[\w'-]>* ) $/ {
        return "/routine/{uri_escape $0}";
    }
    return $_;
}

constant badchars = qw[$ / \ . % ? & = # + " ' : ~ < > |];
my \goodnames = badchars.map: '$' ~ *.uniname.subst(' ', '_', :g);
constant length = badchars.elems;

use experimental :cached;

sub escape-filename($s is copy) is export is cached {
#    return $s if $s ~~ m{^ <[a..z]>+ '://'}; # bail on external links

    loop (my int $i = 0;$i < length;$i++) {
        $s = $s.subst(badchars[$i], goodnames[$i], :g)
    }

    $s
}

sub unescape-percent($s) is cached {
    $s.subst(:g, / [ '%' (<.xdigit> ** 2 ) ]+ /, -> $/ { Buf.new($0.flatmap({ :16(~$_) })).decode('UTF-8') })
}

sub rewrite-url($s) is export is cached {
    my Str $r;
    given $s {
        when / ^ [ 'http' | 'https' | 'irc' ] '://' / {
            # external link, we bail
{'fraction_non_alphanumeric': 0.24927052938724467, 'fraction_numerical': 0.005835764902042517, 'mean_word_length': 2.508771929824561, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1026850', 'n_tokens_mistral': 987, 'n_tokens_neox': 946, 'n_words': 227}