<commit_before>augment class Cool {

    our Int multi method bytes() is export {
        pir::box__PI(pir::bytelength__IS(self))
    }

    # The spec has a more elegant approach for this,
    # but this one works now.
    our Str multi method capitalize() {
        self.lc.split(/\w+/, :all).map({ .Str.ucfirst }).join('');
    }

    our Int multi method chars() is export {
        pir::length__IS(self);
    }

    our multi method chomp() is export {
        # in PIR for speed
        Q:PIR {
            $P0 = find_lex 'self'
            $S0 = $P0
            unless $S0 > '' goto done
            $I0 = ord $S0, -1
            unless $I0 == 10 goto done
            $S0 = chopn $S0, 1
            unless $S0 > '' goto done
            $I0 = ord $S0, -1
            unless $I0 == 13 goto done
            $S0 = chopn $S0, 1
          done:
            %r = new ['Str']
            assign %r, $S0
        }
    }

    multi method subst($matcher, $replacement,
                       :ii(:$samecase), :ss(:$samespace), *%options) {
        my @matches = self.match($matcher, |%options);
        return self unless @matches;
        return self if @matches == 1 && !@matches[0];
        my $prev = 0;
        my $result = '';
        for @matches -> $m {
            $result ~= self.substr($prev, $m.from - $prev);

            my $real_replacement = ~($replacement ~~ Callable ?? $replacement($m) !! $replacement);
            $real_replacement    = $real_replacement.samecase(~$m) if $samecase;
            $real_replacement    = $real_replacement.samespace(~$m) if $samespace;
            $result ~= $real_replacement;
            $prev = $m.to;
        }
        my $last = @matches.pop;
        $result ~= self.substr($last.to);
        $result;
    }

    multi method comb(Regex $matcher = /./, $limit = *, :$match) {
        my $self-string = ~self;
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit;
        gather while $l > 0 && (my $m = $self-string.match($matcher, :c($c))) {
            if $match {
                my $m-clone = $m;
                take $m-clone;
            } else {
                take ~$m;
            }
            $c = $m.to == $c ?? $c + 1 !! $m.to;
            --$l;
        }
    }

    multi method samecase(Cool $pattern) is export {
        my $result = '';
        my $p = '';
        my @pattern = $pattern.comb;
        for self.comb -> $s {
            $p = @pattern.shift if @pattern;
            if $p ~~ /<.upper>/ {
                $result ~= $s.uc;
            } elsif $p ~~ /<.lower>/ {
                $result ~= $s.lc;
            } else {
                $result ~= $s;
            }
        }
        $result;
    }

    method samespace($other as Str) {
        my @pieces = self.split(/\s+/);
        my @new_spaces = $other.comb(/\s+/, @pieces-1);

        my @new = @pieces[0..^@new_spaces] Z @new_spaces;

        if @new_spaces < @pieces-1 {
            my $remainder = self ~~ m:nth(@pieces-1)/\s+/;
            @new.push(self.substr($remainder.from-1));
        }
        else {
            @new.push(@pieces[*-1]);
        }

        return @new.join;
    }


    multi method split(Regex $matcher, $limit = *, :$all) {
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        my $prev-pos = 0;
        return if $l < 0;
        return self.list if $l == 0;
        gather {
            for @.match($matcher, :x(1..$l)) -> $m {
                take self.substr($prev-pos, $m.from - $prev-pos);
                take $m if $all;
                $prev-pos = $m.to;
            }
            take self.substr($prev-pos);
        }
    }

    multi method split($delimiter, $limit = *, :$all) {
        my $match-string = $delimiter.Str;
        return if self eq '' && $delimiter eq '';
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        if $l >= 0 {
            gather {
                while $l-- > 0 {
                    if ($match-string eq "") {
                        last unless $c + 1 < self.chars;
                        take self.substr($c, 1);
                        $c++;
                    } else {
                        my $m = self.index($match-string, $c);
                        last if $m.notdef; # CHEAT, but the best I can do for now
                        take self.substr($c, $m - $c);
                        take $match-string if $all;
                        $c = $m + $match-string.chars;
                    }
                }
                take self.substr($c);
            }
        } else {
            Nil;
        }
    }

    our Str multi method substr($start, $length?) is export {
        my $len = $length // self.chars;
        if ($len < 0) {
            if ($start >= 0) {
                $len += self.chars;
            }
            $len -= $start;
        }

        if ($start > self.chars || $start < -self.chars) {
            return Mu;
        }

        ~pir::substr(self, $start, $len);
    }

    my class LSM {
        has Cool $!source is readonly;
        has      @!substitutions;

        has Int  $!index = 0;
        has Int  $!next_match;
        has      $!next_substitution;
        has      $!substitution_length;

        has Str  $.unsubstituted_text;
        has Str  $.substituted_text;

        method add_substitution($key, $value) {
            push @!substitutions, $key => $value;
        }

        submethod compare_substitution($substitution, Int $pos, Int $length) {
            if $!next_match > $pos
               || $!next_match == $pos && $!substitution_length < $length {

                $!next_match = $pos;
                $!substitution_length = $length;
                $!next_substitution = $substitution;
            }
        }

        multi submethod triage_substitution($_ where { .key ~~ Regex }) {
            my $key = .key;
            return unless $!source.substr($!index) ~~ $key;
            self.compare_substitution($_, $!index + $/.from, $/.to - $/.from);
        }

        multi submethod triage_substitution($_ where { .key ~~ Cool }) {
            return unless defined index($!source, .key, $!index);
            self.compare_substitution($_,
                                      index($!source, .key, $!index),
                                      .key.chars);
        }

        multi submethod triage_substitution($_) {
            die "Don't know how to handle a {.WHAT} as a substitution key";
        }

        multi submethod increment_index(Regex $s) {
            $!source.substr($!index) ~~ $s;
            $!index = $!next_match + $/.chars;
        }

        multi submethod increment_index(Str $s) {
            $!index = $!next_match + $s.chars;
        }

        method next_substitution() {
            $!next_match = $!source.chars;

            for @!substitutions {
                self.triage_substitution($_);
            }

            $!unsubstituted_text
                = $!source.substr($!index, $!next_match - $!index);
            if defined $!next_substitution {
                my $result = $!next_substitution.value;
                $!substituted_text
                    = $result ~~ Callable ?? $result() !! $result;
                self.increment_index($!next_substitution.key);
            }<commit_msg>[core/Cool-str] s/Str/Cool/ inside .trans<commit_after>augment class Cool {

    our Int multi method bytes() is export {
        pir::box__PI(pir::bytelength__IS(self))
    }

    # The spec has a more elegant approach for this,
    # but this one works now.
    our Str multi method capitalize() {
        self.lc.split(/\w+/, :all).map({ .Str.ucfirst }).join('');
    }

    our Int multi method chars() is export {
        pir::length__IS(self);
    }

    our multi method chomp() is export {
        # in PIR for speed
        Q:PIR {
            $P0 = find_lex 'self'
            $S0 = $P0
            unless $S0 > '' goto done
            $I0 = ord $S0, -1
            unless $I0 == 10 goto done
            $S0 = chopn $S0, 1
            unless $S0 > '' goto done
            $I0 = ord $S0, -1
            unless $I0 == 13 goto done
            $S0 = chopn $S0, 1
          done:
            %r = new ['Str']
            assign %r, $S0
        }
    }

    multi method subst($matcher, $replacement,
                       :ii(:$samecase), :ss(:$samespace), *%options) {
        my @matches = self.match($matcher, |%options);
        return self unless @matches;
        return self if @matches == 1 && !@matches[0];
        my $prev = 0;
        my $result = '';
        for @matches -> $m {
            $result ~= self.substr($prev, $m.from - $prev);

            my $real_replacement = ~($replacement ~~ Callable ?? $replacement($m) !! $replacement);
            $real_replacement    = $real_replacement.samecase(~$m) if $samecase;
            $real_replacement    = $real_replacement.samespace(~$m) if $samespace;
            $result ~= $real_replacement;
            $prev = $m.to;
        }
        my $last = @matches.pop;
        $result ~= self.substr($last.to);
        $result;
    }

    multi method comb(Regex $matcher = /./, $limit = *, :$match) {
        my $self-string = ~self;
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit;
        gather while $l > 0 && (my $m = $self-string.match($matcher, :c($c))) {
            if $match {
                my $m-clone = $m;
                take $m-clone;
            } else {
                take ~$m;
            }
            $c = $m.to == $c ?? $c + 1 !! $m.to;
            --$l;
        }
    }

    multi method samecase(Cool $pattern) is export {
        my $result = '';
        my $p = '';
        my @pattern = $pattern.comb;
        for self.comb -> $s {
            $p = @pattern.shift if @pattern;
            if $p ~~ /<.upper>/ {
                $result ~= $s.uc;
            } elsif $p ~~ /<.lower>/ {
                $result ~= $s.lc;
            } else {
                $result ~= $s;
            }
        }
        $result;
    }

    method samespace($other as Str) {
        my @pieces = self.split(/\s+/);
        my @new_spaces = $other.comb(/\s+/, @pieces-1);

        my @new = @pieces[0..^@new_spaces] Z @new_spaces;

        if @new_spaces < @pieces-1 {
            my $remainder = self ~~ m:nth(@pieces-1)/\s+/;
            @new.push(self.substr($remainder.from-1));
        }
        else {
            @new.push(@pieces[*-1]);
        }

        return @new.join;
    }


    multi method split(Regex $matcher, $limit = *, :$all) {
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        my $prev-pos = 0;
        return if $l < 0;
        return self.list if $l == 0;
        gather {
            for @.match($matcher, :x(1..$l)) -> $m {
                take self.substr($prev-pos, $m.from - $prev-pos);
                take $m if $all;
                $prev-pos = $m.to;
            }
            take self.substr($prev-pos);
        }
    }

    multi method split($delimiter, $limit = *, :$all) {
        my $match-string = $delimiter.Str;
        return if self eq '' && $delimiter eq '';
        my $c = 0;
        my $l = $limit ~~ ::Whatever ?? Inf !! $limit - 1;
        if $l >= 0 {
            gather {
                while $l-- > 0 {
                    if ($match-string eq "") {
                        last unless $c + 1 < self.chars;
                        take self.substr($c, 1);
                        $c++;
                    } else {
                        my $m = self.index($match-string, $c);
                        last if $m.notdef; # CHEAT, but the best I can do for now
                        take self.substr($c, $m - $c);
                        take $match-string if $all;
                        $c = $m + $match-string.chars;
                    }
                }
                take self.substr($c);
            }
        } else {
            Nil;
        }
    }

    our Str multi method substr($start, $length?) is export {
        my $len = $length // self.chars;
        if ($len < 0) {
            if ($start >= 0) {
                $len += self.chars;
            }
            $len -= $start;
        }

        if ($start > self.chars || $start < -self.chars) {
            return Mu;
        }

        ~pir::substr(self, $start, $len);
    }

    my class LSM {
        has Cool $!source is readonly;
        has      @!substitutions;

        has Int  $!index = 0;
        has Int  $!next_match;
        has      $!next_substitution;
        has      $!substitution_length;

        has Str  $.unsubstituted_text;
        has Str  $.substituted_text;

        method add_substitution($key, $value) {
            push @!substitutions, $key => $value;
        }

        submethod compare_substitution($substitution, Int $pos, Int $length) {
            if $!next_match > $pos
               || $!next_match == $pos && $!substitution_length < $length {

                $!next_match = $pos;
                $!substitution_length = $length;
                $!next_substitution = $substitution;
            }
        }

        multi submethod triage_substitution($_ where { .key ~~ Regex }) {
            my $key = .key;
            return unless $!source.substr($!index) ~~ $key;
            self.compare_substitution($_, $!index + $/.from, $/.to - $/.from);
        }

        multi submethod triage_substitution($_ where { .key ~~ Cool }) {
            return unless defined index($!source, .key, $!index);
            self.compare_substitution($_,
                                      index($!source, .key, $!index),
                                      .key.chars);
        }

        multi submethod triage_substitution($_) {
            die "Don't know how to handle a {.WHAT} as a substitution key";
        }

        multi submethod increment_index(Regex $s) {
            $!source.substr($!index) ~~ $s;
            $!index = $!next_match + $/.chars;
        }

        multi submethod increment_index(Cool $s) {
            $!index = $!next_match + $s.chars;
        }

        method next_substitution() {
            $!next_match = $!source.chars;

            for @!substitutions {
                self.triage_substitution($_);
            }

            $!unsubstituted_text
                = $!source.substr($!index, $!next_match - $!index);
            if defined $!next_substitution {
                my $result = $!next_substitution.value;
                $!substituted_text
                    = $result ~~ Callable ?? $result() !! $result;
                self.increment_index($!next_substitution.key);
            }
{'fraction_non_alphanumeric': 0.14538015683600408, 'fraction_numerical': 0.006955335833617456, 'mean_word_length': 1.3198354950964883, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2808380', 'n_tokens_mistral': 4661, 'n_tokens_neox': 4407, 'n_words': 1113}