<commit_before>augment class Cool {

    our Int multi method bytes() is export {
        pir::box__PI(pir::bytelength__IS(self))
    }

    # The spec has a more elegant approach for this,
    # but this one works now.
    our Str multi method capitalize() {
        self.lc.split(/\w+/, :all).map({ .Str.ucfirst }).join('');
    }

    our Int multi method chars() is export {
        pir::length__IS(self);
    }

    our multi method chomp() is export {
        if self ~~ /\x0a$/ {
            self.substr(0, self.chars - 1);
        } else {
            self;
        }
    }

    multi method subst($matcher, $replacement,  :g(:$global), :$x) {
        die "Can't combine :g/:global and :x in subst"
            if defined($global) && defined($x);
        my $limit = defined($x) ?? $x +1 !! 2;
        my @chunks = self.split($matcher, :limit($global ?? * !! $limit), :all);
        if defined($x) && (@chunks < 2 * $x) {
            return self;
        }
        loop (my $i = 1; $i < @chunks; $i += 2) {
            pir::store_dynamic_lex__vSP('$/', @chunks[$i]);
            @chunks[$i] = $replacement ~~ Callable ?? $replacement(@chunks[$i]) !! $replacement;
        }
        @chunks.join('');
    }


    multi method comb(Regex $matcher = /./, $limit = *, :$match) {
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit;
        gather while $l > 0 && (my $m = self.match($matcher, :c($c))) {
            take $match ?? $m !! ~$m;
            $c = $m.to == $c ?? $c + 1 !! $m.to;
            --$l;
        }
    }

    multi method samecase($pattern) is export {
        my $result = '';
        my $p = '';
        my @pattern = $pattern.comb;
        for self.comb -> $s {
            $p = @pattern.shift if @pattern;
            if $p ~~ /<.upper>/ {
                $result ~= $s.uc;
            } elsif $p ~~ /<.lower>/ {
                $result ~= $s.lc;
            } else {
                $result ~= $s;
            }
        }
        $result;
    }

    multi method split(Regex $matcher, $limit = *, :$all) {
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        if $l >= 0 {
            gather {
                while $l-- > 0 && (my $m = self.match($matcher, :c($c))) {
                    take self.substr($c, $m.from - $c);
                    take $m if $all;
                    $c = $m.to == $c ?? $c + 1 !! $m.to;
                }
                take self.substr($c);
            }
        } else {
            Nil;
        }
    }

    multi method split($delimiter, $limit = *, :$all) {
        my $match-string = $delimiter.Str;
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        if $l >= 0 {
            gather {
                while $l-- > 0 {
                    if ($match-string eq "") {
                        last unless $c + 1 < self.chars;
                        take self.substr($c, 1);
                        $c++;
                    } else {
                        my $m = self.index($match-string, $c);
                        last if $m.notdef; # CHEAT, but the best I can do for now
                        take self.substr($c, $m - $c);
                        take $match-string if $all;
                        $c = $m + $match-string.chars;
                    }
                }
                take self.substr($c);
            }<commit_msg>Clone the returned match so we can take it without worrying about the next iteration clobbering it.<commit_after>augment class Cool {

    our Int multi method bytes() is export {
        pir::box__PI(pir::bytelength__IS(self))
    }

    # The spec has a more elegant approach for this,
    # but this one works now.
    our Str multi method capitalize() {
        self.lc.split(/\w+/, :all).map({ .Str.ucfirst }).join('');
    }

    our Int multi method chars() is export {
        pir::length__IS(self);
    }

    our multi method chomp() is export {
        if self ~~ /\x0a$/ {
            self.substr(0, self.chars - 1);
        } else {
            self;
        }
    }

    multi method subst($matcher, $replacement,  :g(:$global), :$x) {
        die "Can't combine :g/:global and :x in subst"
            if defined($global) && defined($x);
        my $limit = defined($x) ?? $x +1 !! 2;
        my @chunks = self.split($matcher, :limit($global ?? * !! $limit), :all);
        if defined($x) && (@chunks < 2 * $x) {
            return self;
        }
        loop (my $i = 1; $i < @chunks; $i += 2) {
            pir::store_dynamic_lex__vSP('$/', @chunks[$i]);
            @chunks[$i] = $replacement ~~ Callable ?? $replacement(@chunks[$i]) !! $replacement;
        }
        @chunks.join('');
    }


    multi method comb(Regex $matcher = /./, $limit = *, :$match) {
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit;
        gather while $l > 0 && (my $m = self.match($matcher, :c($c))) {
            take $match ?? $m !! ~$m;
            $c = $m.to == $c ?? $c + 1 !! $m.to;
            --$l;
        }
    }

    multi method samecase($pattern) is export {
        my $result = '';
        my $p = '';
        my @pattern = $pattern.comb;
        for self.comb -> $s {
            $p = @pattern.shift if @pattern;
            if $p ~~ /<.upper>/ {
                $result ~= $s.uc;
            } elsif $p ~~ /<.lower>/ {
                $result ~= $s.lc;
            } else {
                $result ~= $s;
            }
        }
        $result;
    }

    multi method split(Regex $matcher, $limit = *, :$all) {
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        if $l >= 0 {
            gather {
                while $l-- > 0 && (my $m = self.match($matcher, :c($c))) {
                    take self.substr($c, $m.from - $c);
                    my $m-clone = $m;
                    take $m-clone if $all;
                    $c = $m.to == $c ?? $c + 1 !! $m.to;
                }
                take self.substr($c);
            }
        } else {
            Nil;
        }
    }

    multi method split($delimiter, $limit = *, :$all) {
        my $match-string = $delimiter.Str;
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        if $l >= 0 {
            gather {
                while $l-- > 0 {
                    if ($match-string eq "") {
                        last unless $c + 1 < self.chars;
                        take self.substr($c, 1);
                        $c++;
                    } else {
                        my $m = self.index($match-string, $c);
                        last if $m.notdef; # CHEAT, but the best I can do for now
                        take self.substr($c, $m - $c);
                        take $match-string if $all;
                        $c = $m + $match-string.chars;
                    }
                }
                take self.substr($c);
            }
{'fraction_non_alphanumeric': 0.16594454072790293, 'fraction_numerical': 0.006354708261120739, 'mean_word_length': 1.2295556986477785, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5408168', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2142, 'n_words': 593}