<commit_before>unit module Pod::Htmlify;

use URI::Escape;

#| Escape special characters in URLs if necessary
sub url-munge($_) is export {
    return $_ if m{^ <[a..z]>+ '://'};
    return "/type/{uri_escape $_}" if m/^<[A..Z]>/;
    return "/routine/{uri_escape $_}" if m/^<[a..z]>|^<-alpha>*$/;
    # poor man's <identifier>
    if m/ ^ '&'( \w <[[\w'-]>* ) $/ {
        return "/routine/{uri_escape $0}";
    }
    return $_;
}

my \badchars-ntfs = Qw[ / ? < > \ : * | " ¥ ];
my \badchars-unix = Qw[ / ];
my \badchars-url = Qw[ % ];
my \badchars = $*DISTRO.is-win ?? badchars-ntfs !! badchars-unix;
my @badchars = (badchars, badchars-url).flat;
my \goodnames = @badchars.map: '$' ~ *.uniname.subst(' ', '_', :g);
my \length = @badchars.elems;

sub escape-filename($s is copy) is export {
#    return $s if $s ~~ m{^ <[a..z]>+ '://'}; # bail on external links

    loop (my int $i = 0;$i < length;$i++) {
        $s = $s.subst(@badchars[$i], goodnames[$i], :g)
    }

    $s
}

sub unescape-percent($s) {
    $s.subst(:g, / [ '%' (<.xdigit> ** 2 ) ]+ /, -> $/ { Buf.new($0.flatmap({ :16(~$_) })).decode('UTF-8') })
}

sub rewrite-url($s) is export {
    state %cache;
    return %cache{$s} if %cache{$s}:exists;
    my Str $r;
    given $s {
        when / ^ [ 'http' | 'https' | 'irc' ] '://' / {
            # external link, we bail
            return $s;
        }

        when / ^ '#' / {
            # on-page link, we bail
            return $s;
        }
        # Type
        when / ^ <[A..Z]> / {
            $r =  "/type/{escape-filename(unescape-percent($s))}";
            succeed;
        }
        # Routine
        when / ^ <[a..z]> | ^ <-alpha>* $ / {
            $r = "/routine/{escape-filename(unescape-percent($s))}";
            succeed;
        }

        # Special case the really nasty ones
        when / ^ '/routine//' $ /  { return '/routine/' ~ escape-filename('/'); succeed;  }
        when / ^ '/routine///' $ / { return '/routine/' ~ escape-filename('//'); succeed; }

        when / ^
            ([ '/routine/' | '/syntax/' | '/language/' | '/programs/' | '/type/' ]) (<-[#/]>+) [ ('#') (<-[#]>*) ]* $ / {
            $r =  $0 ~ escape-filename(unescape-percent($1)) ~ $2 ~ uri_escape($3);
            succeed;
        }

        default {
            my @parts = $s.split('#');
            $r = escape-filename(@parts[0]) ~ '#' ~ uri_escape(@parts[1]) if @parts[1];
            $r = escape-filename(@parts[0]) unless @parts[1];
        }
    }

    my $file-part = $r.split('#')[0] ~ '.html';
    die "$file-part not found" unless $file-part.IO:e:f:s;
    # URL's can't end with a period. So affix the suffix.
    # If it ends with percent encoded text then we need to add .html to the end too
    $r ~= '.html' if $r.contains('#').not
                  && $r.ends-with( < . >.any)
                  || $r.match: /'%' <:AHex> ** 2 /;
    return %cache{$s} = $r;
}
<commit_msg>Try and fix bug in previous commit<commit_after>unit module Pod::Htmlify;

use URI::Escape;

#| Escape special characters in URLs if necessary
sub url-munge($_) is export {
    return $_ if m{^ <[a..z]>+ '://'};
    return "/type/{uri_escape $_}" if m/^<[A..Z]>/;
    return "/routine/{uri_escape $_}" if m/^<[a..z]>|^<-alpha>*$/;
    # poor man's <identifier>
    if m/ ^ '&'( \w <[[\w'-]>* ) $/ {
        return "/routine/{uri_escape $0}";
    }
    return $_;
}

my \badchars-ntfs = Qw[ / ? < > \ : * | " ¥ ];
my \badchars-unix = Qw[ / ];
my \badchars-url = Qw[ % ];
my \badchars = $*DISTRO.is-win ?? badchars-ntfs !! badchars-unix;
my @badchars = (badchars, badchars-url).flat;
my \goodnames = @badchars.map: '$' ~ *.uniname.subst(' ', '_', :g);
my \length = @badchars.elems;

sub escape-filename($s is copy) is export {
#    return $s if $s ~~ m{^ <[a..z]>+ '://'}; # bail on external links

    loop (my int $i = 0;$i < length;$i++) {
        $s = $s.subst(@badchars[$i], goodnames[$i], :g)
    }

    $s
}

sub unescape-percent($s) {
    $s.subst(:g, / [ '%' (<.xdigit> ** 2 ) ]+ /, -> $/ { Buf.new($0.flatmap({ :16(~$_) })).decode('UTF-8') })
}

sub rewrite-url($s) is export {
    state %cache;
    return %cache{$s} if %cache{$s}:exists;
    my Str $r;
    given $s {
        when / ^ [ 'http' | 'https' | 'irc' ] '://' / {
            # external link, we bail
            return $s;
        }

        when / ^ '#' / {
            # on-page link, we bail
            return $s;
        }
        # Type
        when / ^ <[A..Z]> / {
            $r =  "/type/{escape-filename(unescape-percent($s))}";
            succeed;
        }
        # Routine
        when / ^ <[a..z]> | ^ <-alpha>* $ / {
            $r = "/routine/{escape-filename(unescape-percent($s))}";
            succeed;
        }

        # Special case the really nasty ones
        when / ^ '/routine//' $ /  { return '/routine/' ~ escape-filename('/'); succeed;  }
        when / ^ '/routine///' $ / { return '/routine/' ~ escape-filename('//'); succeed; }

        when / ^
            ([ '/routine/' | '/syntax/' | '/language/' | '/programs/' | '/type/' ]) (<-[#/]>+) [ ('#') (<-[#]>*) ]* $ / {
            $r =  $0 ~ escape-filename(unescape-percent($1)) ~ $2 ~ uri_escape($3);
            succeed;
        }

        default {
            my @parts = $s.split('#');
            $r = escape-filename(@parts[0]) ~ '#' ~ uri_escape(@parts[1]) if @parts[1];
            $r = escape-filename(@parts[0]) unless @parts[1];
        }
    }

    my $file-part = $r.split('#')[0] ~ '.html';
    die "$file-part not found" unless $file-part.IO:e:f:s;
    # URL's can't end with a period. So affix the suffix.
    # If it ends with percent encoded text then we need to add .html to the end too
    if !$r.contains('#') && ( $r.ends-with(<.>) || $r.match: / '%' <:AHex> ** 2 $ / ) {
        $r ~= '.html';
    }
    return %cache{$s} = $r;
}

{'fraction_non_alphanumeric': 0.23583934088568487, 'fraction_numerical': 0.006179196704428424, 'mean_word_length': 1.9018924302788844, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5174163', 'n_tokens_mistral': 2221, 'n_tokens_neox': 2149, 'n_words': 481}