<commit_before>my class NFC is repr('VMArray') is array_type(uint32) { ... }
my class NFD is repr('VMArray') is array_type(uint32) { ... }
my class NFKC is repr('VMArray') is array_type(uint32) { ... }
my class NFKD is repr('VMArray') is array_type(uint32) { ... }

my class Uni does Positional[uint32] does Stringy is repr('VMArray') is array_type(uint32) {

    multi method new(Uni:) { nqp::create(self) }
    multi method new(Uni: *@codes) {
        @codes.elems;  # reify and test for lazy sequences
        my $uni        := nqp::create(self);
        my $codepoints := nqp::getattr(@codes,List,'$!reified');

        nqp::while(
          nqp::elems($codepoints),
          nqp::push_i($uni,nqp::shift($codepoints))
        );

        $uni
    }
    # array[uint32] candidate added in core_epilogue

    my class UniList does PredictiveIterator {
        has $!uni;
        has int $!els;
        has int $!i;
        method !SET-SELF(\uni) {
            $!uni := uni;
            $!i    = -1;
            $!els  = nqp::elems(uni);
            self
        }
        method new (\uni) { nqp::create(self)!SET-SELF: uni }
        method pull-one {
            nqp::islt_i(($!i = nqp::add_i($!i, 1)), $!els)
              ?? nqp::atpos_i($!uni, $!i)
              !! IterationEnd
        }
        method skip-one {
            nqp::islt_i(($!i = nqp::add_i($!i, 1)), $!els)
        }
        method push-all(\target --> IterationEnd) {
            my     $uni := $!uni; # lexicals faster than attrs
            my int $els  = $!els;
            my int $i    = $!i;
            nqp::while(
              nqp::islt_i(($i = nqp::add_i($i, 1)), $els),
              target.push: nqp::atpos_i($uni, $i)
            );
            $!i = $i;
        }
        method count-only(--> Int:D) {
            nqp::p6box_i($!els - $!i - nqp::islt_i($!i,$!els))
        }
        method sink-all(--> IterationEnd) { $!i = $!els }
    }
    method list(Uni:D:) { Seq.new(UniList.new(self)) }

    method Uni(Uni:D:) {
        self
    }

    method NFC(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFC, nqp::create(NFC))
    }

    method NFD(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFD, nqp::create(NFD))
    }

    method NFKC(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFKC, nqp::create(NFKC))
    }

    method NFKD(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFKD, nqp::create(NFKD))
    }

    multi method Str(Uni:D:) {
        nqp::strfromcodes(self)
    }

    multi method Bool(Uni:D:) {
        nqp::hllbool(nqp::elems(self));
    }

    method codes(Uni:D:)   { nqp::elems(self) }
    method elems(Uni:D:)   { nqp::elems(self) }
    method Numeric(Uni:D:) { nqp::elems(self) }
    method Int(Uni:D:)     { nqp::elems(self) }

    multi method ACCEPTS(Uni:D: Uni:D \other --> Bool:D) {
        my $other := nqp::decont(other);
        nqp::hllbool(
          nqp::iseq_i(nqp::elems(self),nqp::elems($other))
            && nqp::stmts(
                 (my int $i = -1),
                 (my int $elems = nqp::elems(self)),
                 nqp::while(
                   nqp::islt_i(($i = nqp::add_i($i,1)),$elems)
                     && nqp::iseq_i(
                          nqp::atpos_i(self,$i),
                          nqp::atpos_i($other,$i)
                        ),
                   nqp::null
                 ),
                 nqp::iseq_i($i,$elems)
               )
        )
    }

    multi method EXISTS-POS(Uni:D: int \pos) {
        nqp::hllbool(
          nqp::islt_i(pos,nqp::elems(self)) && nqp::isge_i(pos,0)
        );
    }
    multi method EXISTS-POS(Uni:D: Int:D \pos) {
        pos < nqp::elems(self) && pos >= 0;
    }

    multi method AT-POS(Uni:D: int \pos) {
        nqp::isge_i(pos,nqp::elems(self)) || nqp::islt_i(pos,0)
          ?? Failure.new(X::OutOfRange.new(
               :what($*INDEX // 'Index'),
               :got(pos),
               :range("0..{nqp::elems(self)-1}")))
          !! nqp::atpos_i(self, pos)
    }
    multi method AT-POS(Uni:D: Int:D \pos) {
        my int $pos = nqp::unbox_i(pos);
        nqp::isge_i($pos,nqp::elems(self)) || nqp::islt_i($pos,0)
          ?? Failure.new(X::OutOfRange.new(
               :what($*INDEX // 'Index'),
               :got(pos),
               :range("0..{nqp::elems(self)-1}")))
          !! nqp::atpos_i(self,$pos)
    }

    multi method gist(Uni:D:) {
        self.^name ~ ':0x<' ~ self.list.fmt('%04x', ' ') ~ '>'
    }

    multi method raku(Uni:D:) {
        'Uni.new(' ~ self.list.fmt('0x%04x', ', ') ~ ')' ~
            (self.WHAT === Uni ?? '' !! '.' ~ self.^name);
    }
}

my class NFD is Uni {
    method new(|) {
        die "Cannot create an NFD directly"; # XXX typed, better message
    }

    method NFD() { self }
}

my class NFC is Uni {
    method new(|) {
        die "Cannot create an NFC directly"; # XXX typed, better message
    }
<commit_msg>Uni \a -> $a optimizations<commit_after>my class NFC is repr('VMArray') is array_type(uint32) { ... }
my class NFD is repr('VMArray') is array_type(uint32) { ... }
my class NFKC is repr('VMArray') is array_type(uint32) { ... }
my class NFKD is repr('VMArray') is array_type(uint32) { ... }

my class Uni does Positional[uint32] does Stringy is repr('VMArray') is array_type(uint32) {

    multi method new(Uni:) { nqp::create(self) }
    multi method new(Uni: *@codes) {
        @codes.elems;  # reify and test for lazy sequences
        my $uni        := nqp::create(self);
        my $codepoints := nqp::getattr(@codes,List,'$!reified');

        nqp::while(
          nqp::elems($codepoints),
          nqp::push_i($uni,nqp::shift($codepoints))
        );

        $uni
    }
    # array[uint32] candidate added in core_epilogue

    my class UniList does PredictiveIterator {
        has $!uni;
        has int $!els;
        has int $!i;
        method !SET-SELF(\uni) {
            $!uni := uni;
            $!i    = -1;
            $!els  = nqp::elems(uni);
            self
        }
        method new (\uni) { nqp::create(self)!SET-SELF: uni }
        method pull-one {
            nqp::islt_i(($!i = nqp::add_i($!i, 1)), $!els)
              ?? nqp::atpos_i($!uni, $!i)
              !! IterationEnd
        }
        method skip-one {
            nqp::islt_i(($!i = nqp::add_i($!i, 1)), $!els)
        }
        method push-all(\target --> IterationEnd) {
            my     $uni := $!uni; # lexicals faster than attrs
            my int $els  = $!els;
            my int $i    = $!i;
            nqp::while(
              nqp::islt_i(($i = nqp::add_i($i, 1)), $els),
              target.push: nqp::atpos_i($uni, $i)
            );
            $!i = $i;
        }
        method count-only(--> Int:D) {
            nqp::p6box_i($!els - $!i - nqp::islt_i($!i,$!els))
        }
        method sink-all(--> IterationEnd) { $!i = $!els }
    }
    method list(Uni:D:) { Seq.new(UniList.new(self)) }

    method Uni(Uni:D:) {
        self
    }

    method NFC(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFC, nqp::create(NFC))
    }

    method NFD(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFD, nqp::create(NFD))
    }

    method NFKC(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFKC, nqp::create(NFKC))
    }

    method NFKD(Uni:D:) {
        nqp::normalizecodes(self, nqp::const::NORMALIZE_NFKD, nqp::create(NFKD))
    }

    multi method Str(Uni:D:) {
        nqp::strfromcodes(self)
    }

    multi method Bool(Uni:D:) {
        nqp::hllbool(nqp::elems(self));
    }

    method codes(Uni:D:)   { nqp::elems(self) }
    method elems(Uni:D:)   { nqp::elems(self) }
    method Numeric(Uni:D:) { nqp::elems(self) }
    method Int(Uni:D:)     { nqp::elems(self) }

    multi method ACCEPTS(Uni:D: Uni:D $other --> Bool:D) {
        nqp::hllbool(
          nqp::iseq_i(nqp::elems(self),nqp::elems($other))
            && nqp::stmts(
                 (my int $i = -1),
                 (my int $elems = nqp::elems(self)),
                 nqp::while(
                   nqp::islt_i(($i = nqp::add_i($i,1)),$elems)
                     && nqp::iseq_i(
                          nqp::atpos_i(self,$i),
                          nqp::atpos_i($other,$i)
                        ),
                   nqp::null
                 ),
                 nqp::iseq_i($i,$elems)
               )
        )
    }

    multi method EXISTS-POS(Uni:D: int $pos) {
        nqp::hllbool(
          nqp::islt_i($pos,nqp::elems(self)) && nqp::isge_i($pos,0)
        );
    }
    multi method EXISTS-POS(Uni:D: Int:D $pos) {
        $pos < nqp::elems(self) && $pos >= 0;
    }

    multi method AT-POS(Uni:D: int $pos) {
        nqp::isge_i($pos,nqp::elems(self)) || nqp::islt_i($pos,0)
          ?? Failure.new(X::OutOfRange.new(
               :what($*INDEX // 'Index'),
               :got($pos),
               :range("0..{nqp::elems(self)-1}")))
          !! nqp::atpos_i(self,$pos)
    }
    multi method AT-POS(Uni:D: Int:D $pos) {
        nqp::isge_i($pos,nqp::elems(self)) || nqp::islt_i($pos,0)
          ?? Failure.new(X::OutOfRange.new(
               :what($*INDEX // 'Index'),
               :got($pos),
               :range("0..{nqp::elems(self)-1}")))
          !! nqp::atpos_i(self,$pos)
    }

    multi method gist(Uni:D:) {
        self.^name ~ ':0x<' ~ self.list.fmt('%04x', ' ') ~ '>'
    }

    multi method raku(Uni:D:) {
        'Uni.new(' ~ self.list.fmt('0x%04x', ', ') ~ ')' ~
            (self.WHAT === Uni ?? '' !! '.' ~ self.^name);
    }
}

my class NFD is Uni {
    method new(|) {
        die "Cannot create an NFD directly"; # XXX typed, better message
    }

    method NFD() { self }
}

my class NFC is Uni {
    method new(|) {
        die "Cannot create an NFC directly"; # XXX typed, better message
    }

{'fraction_non_alphanumeric': 0.1832370528022702, 'fraction_numerical': 0.007094354920441877, 'mean_word_length': 1.8619489559164732, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4182574', 'n_tokens_mistral': 3820, 'n_tokens_neox': 3659, 'n_words': 645}