<commit_before># This class contains generally usable methods creating Iterators.
# There are two reasons for having this in a separate class:
#
# 1. Nice to have a separate file for similar stuff.  Rakudo::Internals
#    has become a hodgepodge of stuff of late.
# 2. Improve readability/searchability of code using these iterators, as
#    many already have a long name, and having them prefixed with the more
#    general Rakudo::Internals in the code, as opposed for the definite
#    Rakudo::Iterator, feels better.

class Rakudo::Iterator {

#-------------------------------------------------------------------------------
# Roles that are used by iterators in the rest of the core settings, in
# alphabetical order for easier perusal.

    # Generic role for iterating over a Blob / Buf.  You need to
    # supply at least a .pull-one.  Takes a Blob / Buf as the only
    # parameter to .new.
    our role Blobby does Iterator {
        has $!blob;
        has Int $!i;   # sadly, this can not be a native int yet  :-(

        method SET-SELF(\blob) {
            nqp::if(
              nqp::isgt_i(nqp::elems(blob),0),
              nqp::stmts(               # something to iterator over
                ($!blob := blob),
                ($!i     = -1),
                self
              ),
              Rakudo::Iterator.Empty    # nothing to iterate
            )
        }
        method new(\blob) { nqp::create(self).SET-SELF(blob) }

        # We can provide a generic push-all to the iterator as the
        # result of a push-all is always immutable, so we can use
        # the atpos_i here in both cases.
        method push-all($target --> IterationEnd) {
            nqp::stmts(
              (my $blob := $!blob),     # attribute access is slower
              (my int $elems = nqp::elems($blob)),
              (my int $i = $!i),
              nqp::while(
                nqp::islt_i(($i = nqp::add_i($i,1)),$elems),
                $target.push(nqp::atpos_i($blob,$i))
              )
            )
        }
        method count-only() { nqp::p6box_i(nqp::elems($!blob)) }
        method sink-all(--> IterationEnd) { $!i = nqp::elems($!blob) }
    }

    # Generic role for iterating over a Map / Hash.  You must
    # at least provide your own .pull-one.  Takes a Map / Hash
    # as the only parameter to .new.
    our role Mappy does Iterator {
        has $!storage;
        has $!iter;

        method SET-SELF(\hash) {
            $!storage := nqp::getattr(hash,Map,'$!storage');
            nqp::if(
              ($!storage.DEFINITE && nqp::elems($!storage)),
              nqp::stmts(              # we have something to iterate over
                ($!iter := nqp::iterator($!storage)),
                self
              ),
              Rakudo::Iterator.Empty   # nothing to iterate
            )
        }
        method new(\hash) { nqp::create(self).SET-SELF(hash) }
        method count-only() { nqp::p6box_i(nqp::elems($!storage)) }
        method bool-only(--> True) { }
        method sink-all(--> IterationEnd) { $!iter := nqp::null }
    }

    # Generic role for iterating over a >1 dimensional shaped list
    # for its lowest branches.  The default .new method takes a List
    # to iterate over.  A consuming class needs to provide a .process
    # method, which will be called with each iteration with the
    # $!indices attribute set to the coordinates of the branch being
    # iterated for this time (with the highest element index set to 0).
    # Consuming class can optionally provide a .done method that will
    # be called just before the iterator returns IterationEnd.
    our role ShapeBranch does Iterator {
        has $!dims;
        has $!indices;
        has Mu $!list;
        has int $!maxdim;
        has int $!maxind;
        has int $!level;

        # Every time process() gets called, the following attributes are set:
        # $!indices  a list_i with current position, with the highest elem 0
        # $!level    level at which exhaustion happened
        # $!dims     a list_i with dimensions
        # $!maxdim   maximum element number in $!dims
        # $!maxind   maximum element number in lowest level list
        method process { ... }           # consumer needs to supply a .process
        method done(--> Nil) { }         # by default no action at end

        method dims() {                  # HLL version of $!dims
            nqp::stmts(
              (my $result := nqp::setelems(nqp::list,nqp::elems($!dims))),
              (my int $i = -1),
              nqp::while(                # convert list_i to list
                nqp::isle_i(($i = nqp::add_i($i,1)),$!maxdim),
                nqp::bindpos($result,$i,nqp::atpos_i($!dims,$i))
              ),
              $result
            )
        }

        method SET-SELF(Mu \list) {
            nqp::stmts(
              nqp::if(
                nqp::istype(list,List),
                nqp::stmts(                                 # List like
                  ($!list := nqp::getattr(list,List,'$!reified')),
                  (my $shape := list.shape),
                  (my int $dims = $shape.elems),     # reifies
                  ($!dims := nqp::setelems(nqp::list_i,$dims)),
                  (my int $i = -1),
                  nqp::while(
                    nqp::islt_i(($i = nqp::add_i($i,1)),$dims),
                    nqp::bindpos_i($!dims,$i,
                      nqp::atpos(nqp::getattr($shape,List,'$!reified'),$i))
                  )
                ),
                ($dims = nqp::elems($!dims := nqp::dimensions($!list := list)))
              ),
              ($!indices := nqp::setelems(nqp::list_i,$dims)),
              ($!maxdim = nqp::sub_i($dims,1)),
              ($!maxind = nqp::sub_i(nqp::atpos_i($!dims,$!maxdim),1)),
              self
            )
        }
        method new(Mu \list) { nqp::create(self).SET-SELF(list) }

        method pull-one() is raw {
            nqp::if(
              nqp::isge_i($!level,0),
              nqp::stmts(                      # still iterating
                (my $result := self.process),  # do the processing
                (my int $level = $!maxdim),
                nqp::until(                    # update indices
                  nqp::islt_i(                 # exhausted ??
                    ($level = nqp::sub_i($level,1)),0) # next level
                    || nqp::stmts(
                    nqp::bindpos_i($!indices,nqp::add_i($level,1),0),  # reset
                    nqp::islt_i(
                      nqp::bindpos_i($!indices,$level, # increment this level
                        nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                      nqp::atpos_i($!dims,$level)      # out of range?
                    ),
                  ),
                  nqp::null
                ),
                ($!level = $level),            # set level for next call
                $result                        # what we found
              ),
              nqp::stmts(
                nqp::if(
                  nqp::iseq_i($!level,-1),
                  nqp::stmts(                  # first time telling we're done
                    self.done,                 # notify we're done
                    ($!level = -2)             # do this only once
                  )
                ),
                IterationEnd                   # done iterating
              )
            )
        }
    }

    # Generic role for iterating over a >1 dimensional shaped list
    # for its values (leaves).  The default .new method takes a List
    # to iterate over.  A consuming class needs to provide a .result
    # method, which will be called with each iteration with the
    # $!indices attribute set to the coordinates of the element being
    # iterated for this time.  In some cases, the iterator is iterated
    # over for the side-effects in .result only.  Which is why this
    # role supplies an optimized .sink-all.
    our role ShapeLeaf does Iterator {
        has $!dims;
        has $!indices;
        has Mu $!list;
        has int $!maxdim;
        has int $!max;

        # Every time .result gets called, the following attributes are set:
        # $!indices  a list_i with current coordinate
        # $!dims     a list_i with dimensions
        # $!maxdim   maximum element number in $!dims
        method result { ... }            # consumer needs to supply a .result

        method indices() {               # HLL version of $!indices
            nqp::stmts(
              (my $result := nqp::setelems(nqp::list,nqp::elems($!indices))),
              (my int $i = -1),
              nqp::while(                # convert list_i to list
                nqp::isle_i(($i = nqp::add_i($i,1)),$!maxdim),
                nqp::bindpos($result,$i,nqp::atpos_i($!indices,$i))
              ),
              $result
            )
        }
        method SET-SELF(Mu \list) {
            nqp::stmts(
              nqp::if(
                nqp::istype(list,List),
                nqp::stmts(                                 # List like
                  ($!list := nqp::getattr(list,List,'$!reified')),
                  (my $shape := list.shape),
                  (my int $dims = $shape.elems),            # reifies
                  ($!dims := nqp::setelems(nqp::list_i,$dims)),
                  (my int $i = -1),
                  nqp::while(
                    nqp::islt_i(($i = nqp::add_i($i,1)),$dims),
                    nqp::bindpos_i($!dims,$i,
                      nqp::atpos(nqp::getattr($shape,List,'$!reified'),$i))
                  )
                ),
                ($dims = nqp::elems($!dims := nqp::dimensions($!list := list)))
              ),
              ($!indices := nqp::setelems(nqp::list_i,$dims)),
              ($!maxdim = nqp::sub_i($dims,1)),
              ($!max    = nqp::atpos_i($!dims,$!maxdim)),
              self
            )
        }
        method new(Mu \list) { nqp::create(self).SET-SELF(list) }

        method pull-one() is raw {
            nqp::if(
              $!indices,
              nqp::stmts(                                 # still iterating
                (my $result := self.result),              # process
                nqp::if(
                  nqp::islt_i(
                    (my int $i =
                      nqp::add_i(nqp::atpos_i($!indices,$!maxdim),1)),
                    $!max
                  ),
                  nqp::bindpos_i($!indices,$!maxdim,$i),  # ready for next
                  nqp::stmts(                             # done for now
                    (my int $level = $!maxdim),
                    nqp::until(                           # update indices
                      nqp::islt_i(                        # exhausted ??
                        ($level = nqp::sub_i($level,1)),0)
                        || nqp::stmts(
                        nqp::bindpos_i($!indices,nqp::add_i($level,1),0),
                        nqp::islt_i(
                          nqp::bindpos_i($!indices,$level,
                            nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                          nqp::atpos_i($!dims,$level)
                        ),
                      ),
                      nqp::null
                    ),
                    nqp::if(
                      nqp::islt_i($level,0),
                      $!indices := nqp::null              # done next time
                    )
                  )
                ),
                $result                                   # what we found
              ),
              IterationEnd                                # done now
            )
        }

        method push-all($target --> IterationEnd) {
            nqp::while(
              $!indices,
              nqp::stmts(                                   # still iterating
                (my int $i = nqp::atpos_i($!indices,$!maxdim)),
                nqp::while(
                  nqp::isle_i(($i = nqp::add_i($i,1)),$!max),
                  nqp::stmts(
                    $target.push(self.result),              # process
                    nqp::bindpos_i($!indices,$!maxdim,$i),  # ready for next
                  )
                ),
                (my int $level = $!maxdim),                 # done for now
                nqp::until(                                 # update indices
                  nqp::islt_i(                              # exhausted ??
                    ($level = nqp::sub_i($level,1)),0)
                    || nqp::stmts(
                    nqp::bindpos_i($!indices,nqp::add_i($level,1),0),
                    nqp::islt_i(
                      nqp::bindpos_i($!indices,$level,
                        nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                      nqp::atpos_i($!dims,$level)
                    ),
                  ),
                  nqp::null
                ),
                nqp::if(
                  nqp::islt_i($level,0),
                  $!indices := nqp::null                    # done
                )
              )
            )
        }

        method sink-all(--> IterationEnd) {
            nqp::while(
              $!indices,
              nqp::stmts(                                   # still iterating
                (my int $i = nqp::atpos_i($!indices,$!maxdim)),
                nqp::while(
                  nqp::isle_i(($i = nqp::add_i($i,1)),$!max),
                  nqp::stmts(
                    self.result,                            # process
                    nqp::bindpos_i($!indices,$!maxdim,$i),  # ready for next
                  )
                ),
                (my int $level = $!maxdim),                 # done for now
                nqp::until(                                 # update indices
                  nqp::islt_i(                              # exhausted ??
                    ($level = nqp::sub_i($level,1)),0)
                    || nqp::stmts(
                    nqp::bindpos_i($!indices,nqp::add_i($level,1),0),
                    nqp::islt_i(
                      nqp::bindpos_i($!indices,$level,
                        nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                      nqp::atpos_i($!dims,$level)
                    ),
                  ),
                  nqp::null
                ),
                nqp::if(
                  nqp::islt_i($level,0),
                  $!indices := nqp::null                    # done
                )
              )
            )
        }
    }

#-------------------------------------------------------------------------------
# Methods that generate an Iterator (in alphabetical order)

    # Return an iterator that will generate a pair with the value as the
    # key and as value the key of the given iterator, basically the
    # .antipairs functionality on 1 dimensional lists.
    method AntiPair(\iterator) {
        class :: does Iterator {
            has Mu $!iter;
            has int $!key;

            method !SET-SELF(\iter) { $!iter := iter; $!key = -1; self }
            method new(\iter) { nqp::create(self)!SET-SELF(iter) }

            method pull-one() is raw {
                nqp::if(
                  nqp::eqaddr((my $pulled := $!iter.pull-one),IterationEnd),
                  IterationEnd,
                  Pair.new($pulled,+($!key = nqp::add_i($!key,1)))
                )
            }
            method push-all($target --> IterationEnd) {
                my $pulled;
                my int $key = -1;
                nqp::until(
                  nqp::eqaddr(($pulled := $!iter.pull-one),IterationEnd),
                  $target.push(Pair.new($pulled,+($key = nqp::add_i($key,1))))
                )
            }
        }.new(iterator)
    }

    # Return an iterator for a given Callable.  The Callable is supposed
    # to return a value for the iterator, or IterationEnd to indicate the
    # data from the Callable is exhausted.  No checks for Slips are done,
    # so they will be passed on as is.  Also optionally takes a flag to
    # mark the iterator as lazy or not: default is False (not lazy)
    proto method Callable(|) { * }
    multi method Callable(&callable) {
        class :: does Iterator {
            has &!callable;
            method new(&callable) {
                nqp::p6bindattrinvres(
                  nqp::create(self),self,'&!callable',&callable)
            }
            method pull-one() is raw { &!callable() }
        }.new(&callable)
    }
    multi method Callable(&callable, Bool() $lazy) {
        nqp::if(
          $lazy,
          class :: does Iterator {
              has &!callable;
              method new(&callable) {
                  nqp::p6bindattrinvres(
                    nqp::create(self),self,'&!callable',&callable)
              }
              method pull-one() is raw { &!callable() }
              method is-lazy(--> True) { }
          }.new(&callable),
          Rakudo::Iterator.Callable(&callable)
        )
    }

    # Return an iterator for a range of 0..^N with a number of elements.
    # The third parameter indicates whether an IterationBuffer should be
    # returned (1) for each combinatin, or a fully reified List (0).
    # Has a highly optimized count-only, for those cases when one is only
    # interested in the number of combinations, rather than the actual
    # combinations.  The workhorse of combinations().
    method Combinations($n, $k, int $b) {
        nqp::if(
          $n > 0 && nqp::isbig_I(nqp::decont($n)),    # must be HLL comparison
          X::OutOfRange.new(
            :what("First parameter"),
            :got($n),
            :range("-Inf^..{$*KERNEL.bits == 32 ?? 2**28-1 !! 2**31-1}")
          ).throw,
          nqp::if(
            # k = 0 → can pick just 1 combination (empty list); return ((),)
            $k == 0,                                  # Must be HLL comparison
            Rakudo::Iterator.OneValue(
              nqp::create(nqp::if($b,IterationBuffer,List))
            ),
            nqp::if(
              # n < 1 → we have an empty list to pick from
              # n < k → not enough items to pick combination of k items
              $n < 1 || $n < $k || $k < 0,            # must be HLL comparisons
              Rakudo::Iterator.Empty,                 # nothing to return
              class :: does Iterator {
                  has int $!n;
                  has int $!k;
                  has int $!b;
                  has Mu $!stack;
                  has Mu $!combination;
                  method !SET-SELF(\n,\k,\b) {
                      nqp::stmts(
                        ($!n = n),
                        ($!k = k),
                        ($!b = b),
                        ($!stack := nqp::list_i(0)),
                        ($!combination := nqp::create(IterationBuffer)),
                        self
                    )
                  }
                  method new(\n,\k,\b) { nqp::create(self)!SET-SELF(n,k,b) }

                  method pull-one() {
                      nqp::stmts(
                        (my int $n = $!n),          # lexicals faster
                        (my int $k = $!k),
                        (my int $running = 1),
                        nqp::while(
                          ($running && (my int $elems = nqp::elems($!stack))),
                          nqp::stmts(
                            (my int $index = nqp::sub_i($elems,1)),
                            (my int $value = nqp::pop_i($!stack)),
                            nqp::while(
                              (nqp::islt_i($value,$n)
                                && nqp::islt_i($index,$k)),
                              nqp::stmts(
                                nqp::bindpos($!combination,
                                  $index,nqp::clone($value)),
                                ($index = nqp::add_i($index,1)),
                                ($value = nqp::add_i($value,1)),
                                nqp::push_i($!stack,$value)
                              )
                            ),
                            ($running = nqp::isne_i($index,$k)),
                          )
                        ),
                        nqp::if(
                          nqp::iseq_i($index,$k),
                          nqp::if(
                            $b,
                            nqp::clone($!combination),
                            nqp::p6bindattrinvres(
                              nqp::create(List),List,'$!reified',
                              nqp::clone($!combination)
                            )
                          ),
                          IterationEnd
                        )
                      )
                  }
                  method count-only {
                      ([*] ($!n ... 0) Z/ 1 .. min($!n - $!k, $!k)).Int
                  }
                  method bool-only(--> True) { }
              }.new($n,$k,$b)
            )
          )
        )
    }

    # Return an iterator that will cross the given iterables (with &[,])
    # Basically the functionality of @a X @b
    method CrossIterables(@iterables) {
        nqp::if(
          nqp::isgt_i((my int $n = @iterables.elems),1),  # reifies

          # actually need to do some crossing (probably)
          class :: does Iterator {
              has $!iterators;  # iterator per iterable, if any
              has $!reifieds;   # cached values (either complete, or so far)
              has $!indices;    # indices of virtual matrix of crossed values
              has $!next;       # IterationBuffer with next values to return
              has int $!lazy;   # whether the outer iterator is lazy
              has int $!top;    # index of top reified/iterator

              method !SET-SELF(\iterables) {
                  nqp::stmts(
                    (my $iterables := nqp::getattr(iterables,List,'$!reified')),
                    (my int $elems  = nqp::elems($iterables)),
                    ($!iterators   := nqp::setelems(nqp::list,$elems)),
                    ($!reifieds    := nqp::setelems(nqp::list,$elems)),
                    ($!next :=
                      nqp::setelems(nqp::create(IterationBuffer),$elems)),

                    # loop over all iterables
                    (my int $i = -1),
                    nqp::while(
                      nqp::islt_i(($i = nqp::add_i($i,1)),$elems),

                      # set up initial value of index $i with...
                      nqp::bindpos($!next,$i,nqp::if(
                        nqp::iscont(my $elem := nqp::atpos($iterables,$i)),

                        # single value same as reified list of 1
                        nqp::bindpos(
                          nqp::bindpos($!reifieds,$i,nqp::list),
                          0,
                          $elem
                        ),

                        # something more elaborate
                        nqp::if(
                          nqp::istype($elem,List)
                            && nqp::not_i(nqp::isnull(
                                 nqp::getattr($elem,List,'$!todo'))),

                          # it's a List, may have a reified we can use directly
                          nqp::if(
                            nqp::isnull(
                              $elem := nqp::getattr($elem,List,'$!reified'))
                              || nqp::iseq_i(nqp::elems($elem),0),

                            # cross with an empty list is always an empty list
                            (return Rakudo::Iterator.EmptyIterator),

                            # use the available reified directly
                            nqp::stmts(
                              nqp::bindpos($!reifieds,$i,$elem),
                              nqp::atpos($elem,0)
                            )
                          ),

                          # need to set up an iterator
                          nqp::stmts(
                            nqp::if($elem.is-lazy,($!lazy = 1)),
                            nqp::if(
                              nqp::eqaddr(
                                (my $pulled :=
                                  ($elem := $elem.iterator).pull-one),
                                IterationEnd
                              ),

                              # cross with an empty list is an empty list
                              (return Rakudo::Iterator.EmptyIterator),

                              # set up the iterator stuff
                              nqp::stmts(
                                nqp::bindpos($!iterators,$i,$elem),
                                nqp::bindpos($!reifieds,$i,nqp::list($pulled)),
                                $pulled
                              )
                            )
                          )
                        )
                      ))
                    ),

                    # indices start with 0 xx $elems
                    ($!indices := nqp::setelems(nqp::list_i,$elems)),
                    ($!top = nqp::sub_i($elems,1)),
                    self
                  )
              }
              method new(\iterables) { nqp::create(self)!SET-SELF(iterables) }
              method pull-one() {
                  nqp::if(
                    nqp::isnull($!next),
                    IterationEnd,
                    nqp::stmts(

                      # set up result of this pull
                      (my $result := nqp::p6bindattrinvres(
                        nqp::create(List),List,'$!reified',nqp::clone($!next))),

                      # start working on next result
                      nqp::unless(
                        nqp::isnull(nqp::atpos($!iterators,$!top)),

                        # top level is still iterator, fetch
                        nqp::if(
                          nqp::eqaddr(
                            (my $pulled :=
                              nqp::atpos($!iterators,$!top).pull-one),
                            IterationEnd
                          ),
                          # iterator no more
                          nqp::bindpos($!iterators,$!top,nqp::null),

                          # push value, let normal reifier handler handle
                          nqp::push(
                            nqp::atpos($!reifieds,$!top),
                            $pulled
                          )
                        )
                      ),

                      # no iterator, must use reified list
                      nqp::if(
                        nqp::islt_i(
                          (my int $index =
                            nqp::add_i(nqp::atpos_i($!indices,$!top),1)),
                          nqp::elems(nqp::atpos($!reifieds,$!top))
                        ),

                        # within range, update next result and index
                        nqp::bindpos($!next,$!top,
                          nqp::atpos(
                            nqp::atpos($!reifieds,$!top),
                            nqp::bindpos_i($!indices,$!top,$index)
                          )
                        ),

                        # need to update lower levels
                        nqp::stmts(

                          # update topmost value (go back to first)
                          nqp::bindpos($!next,$!top,
                            nqp::atpos(
                              nqp::atpos($!reifieds,$!top),
                              nqp::bindpos_i($!indices,$!top,0)
                            )
                          ),

                          # until we're at the bottom
                          (my int $level = $!top),
                          nqp::while(
                            nqp::isge_i(($level = nqp::sub_i($level,1)),0),
                            nqp::if(
                              nqp::isnull(nqp::atpos($!iterators,$level)),

                              # can use reified list at this level
                              nqp::if(
                                nqp::islt_i(
                                  ($index = nqp::add_i(
                                    nqp::atpos_i($!indices,$level),1)),
                                  nqp::elems(nqp::atpos($!reifieds,$level))
                                ),

                                # within range, update next result and index
                                nqp::stmts(
                                  nqp::bindpos($!next,$level,
                                    nqp::atpos(
                                      nqp::atpos($!reifieds,$level),
                                      nqp::bindpos_i($!indices,$level,$index)
                                    )
                                  ),
                                  ($level = -1)  # done searching
                                ),

                                # reset this level
                                nqp::bindpos($!next,$level,
                                  nqp::atpos(
                                    nqp::atpos($!reifieds,$level),
                                    nqp::bindpos_i($!indices,$level,0)
                                  )
                                )
                              ),

                              # still an iterator at this level
                              nqp::if(
                                nqp::eqaddr(
                                  ($pulled :=
                                    nqp::atpos($!iterators,$level).pull-one),
                                  IterationEnd
                                ),

                                # exhausted iterator, reset to reified
                                nqp::stmts(
                                  nqp::bindpos($!iterators,$level,nqp::null),
                                  nqp::bindpos($!next,$level,
                                    nqp::atpos(
                                      nqp::atpos($!reifieds,$level),
                                      nqp::bindpos_i($!indices,$level,0)
                                    )
                                  )
                                ),

                                # new value, add to reified, update indices
                                nqp::stmts(
                                  nqp::bindpos(
                                    $!next,
                                    $level,
                                  nqp::bindpos(
                                      nqp::atpos($!reifieds,$level),
                                      nqp::bindpos_i(
                                        $!indices,
                                        $level,
                                        nqp::add_i(
                                          nqp::atpos_i($!indices,$level),
                                          1
                                        )
                                      ),
                                      $pulled
                                    )
                                  ),
                                  ($level = -1)  # done searching
                                )
                            )
                            )
                          ),
                          nqp::if(
                            nqp::iseq_i($level,-1),

                            # was last iteration, free up everything now
                            ($!next :=
                              $!iterators := $!reifieds := $!indices :=
                              nqp::null)
                          )
                        )
                      ),
                      $result
                    )
                  )
              }
              method is-lazy() { nqp::p6bool($!lazy) }
          }.new(@iterables),

          # simpler cases
          nqp::if(
            nqp::iseq_i($n,0),
            # nothing to cross, so return an empty list
            Rakudo::Iterator.Empty,
            # only 1 list to cross, which is the list itself
            nqp::atpos(nqp::getattr(@iterables,List,'$!reified'),0).iterator
          )
        )
    }

    # Return an iterator that will cross the given iterables and map
    # the result with the given mapper Callable.  Basically the
    # functionality of @a Xop @b (with the op  -> mapper functionality
    # to be supplied externally).
    method CrossIterablesMap(@iterables,&mapper) {
        nqp::if(
          nqp::isgt_i((my int $n = @iterables.elems),1),  # reifies

          # actually need to do some crossing (probably)
          class :: does Iterator {
              has $!iterators;  # iterator per iterable, if any
              has $!reifieds;   # cached values (either complete, or so far)
              has $!indices;    # indices of virtual matrix of crossed values
              has $!next;       # IterationBuffer with next values to return
              has $!mapper;     # Callable to do final result mapping
              has int $!lazy;   # whether the outer iterator is lazy
              has int $!top;    # index of top reified/iterator

              method !SET-SELF(\iterables,\mapper) {
                  nqp::stmts(
                    (my $iterables := nqp::getattr(iterables,List,'$!reified')),
                    (my int $elems  = nqp::elems($iterables)),
                    ($!iterators   := nqp::setelems(nqp::list,$elems)),
                    ($!reifieds    := nqp::setelems(nqp::list,$elems)),
                    ($!next :=
                      nqp::setelems(nqp::create(IterationBuffer),$elems)),

                    # loop over all iterables
                    (my int $i = -1),
                    nqp::while(
                      nqp::islt_i(($i = nqp::add_i($i,1)),$elems),

                      # set up initial value of index $i with...
                      nqp::bindpos($!next,$i,nqp::if(
                        nqp::iscont(my $elem := nqp::atpos($iterables,$i)),

                        # single value same as reified list of 1
                        nqp::bindpos(
                          nqp::bindpos($!reifieds,$i,nqp::list),
                          0,
                          $elem
                        ),

                        # something more elaborate
                        nqp::if(
                          nqp::istype($elem,List)
                            && nqp::not_i(nqp::isnull(<commit_msg>Make 1 X foo about 20% faster<commit_after># This class contains generally usable methods creating Iterators.
# There are two reasons for having this in a separate class:
#
# 1. Nice to have a separate file for similar stuff.  Rakudo::Internals
#    has become a hodgepodge of stuff of late.
# 2. Improve readability/searchability of code using these iterators, as
#    many already have a long name, and having them prefixed with the more
#    general Rakudo::Internals in the code, as opposed for the definite
#    Rakudo::Iterator, feels better.

class Rakudo::Iterator {

#-------------------------------------------------------------------------------
# Roles that are used by iterators in the rest of the core settings, in
# alphabetical order for easier perusal.

    # Generic role for iterating over a Blob / Buf.  You need to
    # supply at least a .pull-one.  Takes a Blob / Buf as the only
    # parameter to .new.
    our role Blobby does Iterator {
        has $!blob;
        has Int $!i;   # sadly, this can not be a native int yet  :-(

        method SET-SELF(\blob) {
            nqp::if(
              nqp::isgt_i(nqp::elems(blob),0),
              nqp::stmts(               # something to iterator over
                ($!blob := blob),
                ($!i     = -1),
                self
              ),
              Rakudo::Iterator.Empty    # nothing to iterate
            )
        }
        method new(\blob) { nqp::create(self).SET-SELF(blob) }

        # We can provide a generic push-all to the iterator as the
        # result of a push-all is always immutable, so we can use
        # the atpos_i here in both cases.
        method push-all($target --> IterationEnd) {
            nqp::stmts(
              (my $blob := $!blob),     # attribute access is slower
              (my int $elems = nqp::elems($blob)),
              (my int $i = $!i),
              nqp::while(
                nqp::islt_i(($i = nqp::add_i($i,1)),$elems),
                $target.push(nqp::atpos_i($blob,$i))
              )
            )
        }
        method count-only() { nqp::p6box_i(nqp::elems($!blob)) }
        method sink-all(--> IterationEnd) { $!i = nqp::elems($!blob) }
    }

    # Generic role for iterating over a Map / Hash.  You must
    # at least provide your own .pull-one.  Takes a Map / Hash
    # as the only parameter to .new.
    our role Mappy does Iterator {
        has $!storage;
        has $!iter;

        method SET-SELF(\hash) {
            $!storage := nqp::getattr(hash,Map,'$!storage');
            nqp::if(
              ($!storage.DEFINITE && nqp::elems($!storage)),
              nqp::stmts(              # we have something to iterate over
                ($!iter := nqp::iterator($!storage)),
                self
              ),
              Rakudo::Iterator.Empty   # nothing to iterate
            )
        }
        method new(\hash) { nqp::create(self).SET-SELF(hash) }
        method count-only() { nqp::p6box_i(nqp::elems($!storage)) }
        method bool-only(--> True) { }
        method sink-all(--> IterationEnd) { $!iter := nqp::null }
    }

    # Generic role for iterating over a >1 dimensional shaped list
    # for its lowest branches.  The default .new method takes a List
    # to iterate over.  A consuming class needs to provide a .process
    # method, which will be called with each iteration with the
    # $!indices attribute set to the coordinates of the branch being
    # iterated for this time (with the highest element index set to 0).
    # Consuming class can optionally provide a .done method that will
    # be called just before the iterator returns IterationEnd.
    our role ShapeBranch does Iterator {
        has $!dims;
        has $!indices;
        has Mu $!list;
        has int $!maxdim;
        has int $!maxind;
        has int $!level;

        # Every time process() gets called, the following attributes are set:
        # $!indices  a list_i with current position, with the highest elem 0
        # $!level    level at which exhaustion happened
        # $!dims     a list_i with dimensions
        # $!maxdim   maximum element number in $!dims
        # $!maxind   maximum element number in lowest level list
        method process { ... }           # consumer needs to supply a .process
        method done(--> Nil) { }         # by default no action at end

        method dims() {                  # HLL version of $!dims
            nqp::stmts(
              (my $result := nqp::setelems(nqp::list,nqp::elems($!dims))),
              (my int $i = -1),
              nqp::while(                # convert list_i to list
                nqp::isle_i(($i = nqp::add_i($i,1)),$!maxdim),
                nqp::bindpos($result,$i,nqp::atpos_i($!dims,$i))
              ),
              $result
            )
        }

        method SET-SELF(Mu \list) {
            nqp::stmts(
              nqp::if(
                nqp::istype(list,List),
                nqp::stmts(                                 # List like
                  ($!list := nqp::getattr(list,List,'$!reified')),
                  (my $shape := list.shape),
                  (my int $dims = $shape.elems),     # reifies
                  ($!dims := nqp::setelems(nqp::list_i,$dims)),
                  (my int $i = -1),
                  nqp::while(
                    nqp::islt_i(($i = nqp::add_i($i,1)),$dims),
                    nqp::bindpos_i($!dims,$i,
                      nqp::atpos(nqp::getattr($shape,List,'$!reified'),$i))
                  )
                ),
                ($dims = nqp::elems($!dims := nqp::dimensions($!list := list)))
              ),
              ($!indices := nqp::setelems(nqp::list_i,$dims)),
              ($!maxdim = nqp::sub_i($dims,1)),
              ($!maxind = nqp::sub_i(nqp::atpos_i($!dims,$!maxdim),1)),
              self
            )
        }
        method new(Mu \list) { nqp::create(self).SET-SELF(list) }

        method pull-one() is raw {
            nqp::if(
              nqp::isge_i($!level,0),
              nqp::stmts(                      # still iterating
                (my $result := self.process),  # do the processing
                (my int $level = $!maxdim),
                nqp::until(                    # update indices
                  nqp::islt_i(                 # exhausted ??
                    ($level = nqp::sub_i($level,1)),0) # next level
                    || nqp::stmts(
                    nqp::bindpos_i($!indices,nqp::add_i($level,1),0),  # reset
                    nqp::islt_i(
                      nqp::bindpos_i($!indices,$level, # increment this level
                        nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                      nqp::atpos_i($!dims,$level)      # out of range?
                    ),
                  ),
                  nqp::null
                ),
                ($!level = $level),            # set level for next call
                $result                        # what we found
              ),
              nqp::stmts(
                nqp::if(
                  nqp::iseq_i($!level,-1),
                  nqp::stmts(                  # first time telling we're done
                    self.done,                 # notify we're done
                    ($!level = -2)             # do this only once
                  )
                ),
                IterationEnd                   # done iterating
              )
            )
        }
    }

    # Generic role for iterating over a >1 dimensional shaped list
    # for its values (leaves).  The default .new method takes a List
    # to iterate over.  A consuming class needs to provide a .result
    # method, which will be called with each iteration with the
    # $!indices attribute set to the coordinates of the element being
    # iterated for this time.  In some cases, the iterator is iterated
    # over for the side-effects in .result only.  Which is why this
    # role supplies an optimized .sink-all.
    our role ShapeLeaf does Iterator {
        has $!dims;
        has $!indices;
        has Mu $!list;
        has int $!maxdim;
        has int $!max;

        # Every time .result gets called, the following attributes are set:
        # $!indices  a list_i with current coordinate
        # $!dims     a list_i with dimensions
        # $!maxdim   maximum element number in $!dims
        method result { ... }            # consumer needs to supply a .result

        method indices() {               # HLL version of $!indices
            nqp::stmts(
              (my $result := nqp::setelems(nqp::list,nqp::elems($!indices))),
              (my int $i = -1),
              nqp::while(                # convert list_i to list
                nqp::isle_i(($i = nqp::add_i($i,1)),$!maxdim),
                nqp::bindpos($result,$i,nqp::atpos_i($!indices,$i))
              ),
              $result
            )
        }
        method SET-SELF(Mu \list) {
            nqp::stmts(
              nqp::if(
                nqp::istype(list,List),
                nqp::stmts(                                 # List like
                  ($!list := nqp::getattr(list,List,'$!reified')),
                  (my $shape := list.shape),
                  (my int $dims = $shape.elems),            # reifies
                  ($!dims := nqp::setelems(nqp::list_i,$dims)),
                  (my int $i = -1),
                  nqp::while(
                    nqp::islt_i(($i = nqp::add_i($i,1)),$dims),
                    nqp::bindpos_i($!dims,$i,
                      nqp::atpos(nqp::getattr($shape,List,'$!reified'),$i))
                  )
                ),
                ($dims = nqp::elems($!dims := nqp::dimensions($!list := list)))
              ),
              ($!indices := nqp::setelems(nqp::list_i,$dims)),
              ($!maxdim = nqp::sub_i($dims,1)),
              ($!max    = nqp::atpos_i($!dims,$!maxdim)),
              self
            )
        }
        method new(Mu \list) { nqp::create(self).SET-SELF(list) }

        method pull-one() is raw {
            nqp::if(
              $!indices,
              nqp::stmts(                                 # still iterating
                (my $result := self.result),              # process
                nqp::if(
                  nqp::islt_i(
                    (my int $i =
                      nqp::add_i(nqp::atpos_i($!indices,$!maxdim),1)),
                    $!max
                  ),
                  nqp::bindpos_i($!indices,$!maxdim,$i),  # ready for next
                  nqp::stmts(                             # done for now
                    (my int $level = $!maxdim),
                    nqp::until(                           # update indices
                      nqp::islt_i(                        # exhausted ??
                        ($level = nqp::sub_i($level,1)),0)
                        || nqp::stmts(
                        nqp::bindpos_i($!indices,nqp::add_i($level,1),0),
                        nqp::islt_i(
                          nqp::bindpos_i($!indices,$level,
                            nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                          nqp::atpos_i($!dims,$level)
                        ),
                      ),
                      nqp::null
                    ),
                    nqp::if(
                      nqp::islt_i($level,0),
                      $!indices := nqp::null              # done next time
                    )
                  )
                ),
                $result                                   # what we found
              ),
              IterationEnd                                # done now
            )
        }

        method push-all($target --> IterationEnd) {
            nqp::while(
              $!indices,
              nqp::stmts(                                   # still iterating
                (my int $i = nqp::atpos_i($!indices,$!maxdim)),
                nqp::while(
                  nqp::isle_i(($i = nqp::add_i($i,1)),$!max),
                  nqp::stmts(
                    $target.push(self.result),              # process
                    nqp::bindpos_i($!indices,$!maxdim,$i),  # ready for next
                  )
                ),
                (my int $level = $!maxdim),                 # done for now
                nqp::until(                                 # update indices
                  nqp::islt_i(                              # exhausted ??
                    ($level = nqp::sub_i($level,1)),0)
                    || nqp::stmts(
                    nqp::bindpos_i($!indices,nqp::add_i($level,1),0),
                    nqp::islt_i(
                      nqp::bindpos_i($!indices,$level,
                        nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                      nqp::atpos_i($!dims,$level)
                    ),
                  ),
                  nqp::null
                ),
                nqp::if(
                  nqp::islt_i($level,0),
                  $!indices := nqp::null                    # done
                )
              )
            )
        }

        method sink-all(--> IterationEnd) {
            nqp::while(
              $!indices,
              nqp::stmts(                                   # still iterating
                (my int $i = nqp::atpos_i($!indices,$!maxdim)),
                nqp::while(
                  nqp::isle_i(($i = nqp::add_i($i,1)),$!max),
                  nqp::stmts(
                    self.result,                            # process
                    nqp::bindpos_i($!indices,$!maxdim,$i),  # ready for next
                  )
                ),
                (my int $level = $!maxdim),                 # done for now
                nqp::until(                                 # update indices
                  nqp::islt_i(                              # exhausted ??
                    ($level = nqp::sub_i($level,1)),0)
                    || nqp::stmts(
                    nqp::bindpos_i($!indices,nqp::add_i($level,1),0),
                    nqp::islt_i(
                      nqp::bindpos_i($!indices,$level,
                        nqp::add_i(nqp::atpos_i($!indices,$level),1)),
                      nqp::atpos_i($!dims,$level)
                    ),
                  ),
                  nqp::null
                ),
                nqp::if(
                  nqp::islt_i($level,0),
                  $!indices := nqp::null                    # done
                )
              )
            )
        }
    }

#-------------------------------------------------------------------------------
# Methods that generate an Iterator (in alphabetical order)

    # Return an iterator that will generate a pair with the value as the
    # key and as value the key of the given iterator, basically the
    # .antipairs functionality on 1 dimensional lists.
    method AntiPair(\iterator) {
        class :: does Iterator {
            has Mu $!iter;
            has int $!key;

            method !SET-SELF(\iter) { $!iter := iter; $!key = -1; self }
            method new(\iter) { nqp::create(self)!SET-SELF(iter) }

            method pull-one() is raw {
                nqp::if(
                  nqp::eqaddr((my $pulled := $!iter.pull-one),IterationEnd),
                  IterationEnd,
                  Pair.new($pulled,+($!key = nqp::add_i($!key,1)))
                )
            }
            method push-all($target --> IterationEnd) {
                my $pulled;
                my int $key = -1;
                nqp::until(
                  nqp::eqaddr(($pulled := $!iter.pull-one),IterationEnd),
                  $target.push(Pair.new($pulled,+($key = nqp::add_i($key,1))))
                )
            }
        }.new(iterator)
    }

    # Return an iterator for a given Callable.  The Callable is supposed
    # to return a value for the iterator, or IterationEnd to indicate the
    # data from the Callable is exhausted.  No checks for Slips are done,
    # so they will be passed on as is.  Also optionally takes a flag to
    # mark the iterator as lazy or not: default is False (not lazy)
    proto method Callable(|) { * }
    multi method Callable(&callable) {
        class :: does Iterator {
            has &!callable;
            method new(&callable) {
                nqp::p6bindattrinvres(
                  nqp::create(self),self,'&!callable',&callable)
            }
            method pull-one() is raw { &!callable() }
        }.new(&callable)
    }
    multi method Callable(&callable, Bool() $lazy) {
        nqp::if(
          $lazy,
          class :: does Iterator {
              has &!callable;
              method new(&callable) {
                  nqp::p6bindattrinvres(
                    nqp::create(self),self,'&!callable',&callable)
              }
              method pull-one() is raw { &!callable() }
              method is-lazy(--> True) { }
          }.new(&callable),
          Rakudo::Iterator.Callable(&callable)
        )
    }

    # Return an iterator for a range of 0..^N with a number of elements.
    # The third parameter indicates whether an IterationBuffer should be
    # returned (1) for each combinatin, or a fully reified List (0).
    # Has a highly optimized count-only, for those cases when one is only
    # interested in the number of combinations, rather than the actual
    # combinations.  The workhorse of combinations().
    method Combinations($n, $k, int $b) {
        nqp::if(
          $n > 0 && nqp::isbig_I(nqp::decont($n)),    # must be HLL comparison
          X::OutOfRange.new(
            :what("First parameter"),
            :got($n),
            :range("-Inf^..{$*KERNEL.bits == 32 ?? 2**28-1 !! 2**31-1}")
          ).throw,
          nqp::if(
            # k = 0 → can pick just 1 combination (empty list); return ((),)
            $k == 0,                                  # Must be HLL comparison
            Rakudo::Iterator.OneValue(
              nqp::create(nqp::if($b,IterationBuffer,List))
            ),
            nqp::if(
              # n < 1 → we have an empty list to pick from
              # n < k → not enough items to pick combination of k items
              $n < 1 || $n < $k || $k < 0,            # must be HLL comparisons
              Rakudo::Iterator.Empty,                 # nothing to return
              class :: does Iterator {
                  has int $!n;
                  has int $!k;
                  has int $!b;
                  has Mu $!stack;
                  has Mu $!combination;
                  method !SET-SELF(\n,\k,\b) {
                      nqp::stmts(
                        ($!n = n),
                        ($!k = k),
                        ($!b = b),
                        ($!stack := nqp::list_i(0)),
                        ($!combination := nqp::create(IterationBuffer)),
                        self
                    )
                  }
                  method new(\n,\k,\b) { nqp::create(self)!SET-SELF(n,k,b) }

                  method pull-one() {
                      nqp::stmts(
                        (my int $n = $!n),          # lexicals faster
                        (my int $k = $!k),
                        (my int $running = 1),
                        nqp::while(
                          ($running && (my int $elems = nqp::elems($!stack))),
                          nqp::stmts(
                            (my int $index = nqp::sub_i($elems,1)),
                            (my int $value = nqp::pop_i($!stack)),
                            nqp::while(
                              (nqp::islt_i($value,$n)
                                && nqp::islt_i($index,$k)),
                              nqp::stmts(
                                nqp::bindpos($!combination,
                                  $index,nqp::clone($value)),
                                ($index = nqp::add_i($index,1)),
                                ($value = nqp::add_i($value,1)),
                                nqp::push_i($!stack,$value)
                              )
                            ),
                            ($running = nqp::isne_i($index,$k)),
                          )
                        ),
                        nqp::if(
                          nqp::iseq_i($index,$k),
                          nqp::if(
                            $b,
                            nqp::clone($!combination),
                            nqp::p6bindattrinvres(
                              nqp::create(List),List,'$!reified',
                              nqp::clone($!combination)
                            )
                          ),
                          IterationEnd
                        )
                      )
                  }
                  method count-only {
                      ([*] ($!n ... 0) Z/ 1 .. min($!n - $!k, $!k)).Int
                  }
                  method bool-only(--> True) { }
              }.new($n,$k,$b)
            )
          )
        )
    }

    # Return an iterator that will cross the given iterables (with &[,])
    # Basically the functionality of @a X @b
    method CrossIterables(@iterables) {
        nqp::if(
          nqp::isgt_i((my int $n = @iterables.elems),1),  # reifies

          # actually need to do some crossing (probably)
          class :: does Iterator {
              has $!iterators;  # iterator per iterable, if any
              has $!reifieds;   # cached values (either complete, or so far)
              has $!indices;    # indices of virtual matrix of crossed values
              has $!next;       # IterationBuffer with next values to return
              has int $!lazy;   # whether the outer iterator is lazy
              has int $!top;    # index of top reified/iterator

              method !SET-SELF(\iterables) {
                  nqp::stmts(
                    (my $iterables := nqp::getattr(iterables,List,'$!reified')),
                    (my int $elems  = nqp::elems($iterables)),
                    ($!iterators   := nqp::setelems(nqp::list,$elems)),
                    ($!reifieds    := nqp::setelems(nqp::list,$elems)),
                    ($!next :=
                      nqp::setelems(nqp::create(IterationBuffer),$elems)),

                    # loop over all iterables
                    (my int $i = -1),
                    nqp::while(
                      nqp::islt_i(($i = nqp::add_i($i,1)),$elems),

                      # set up initial value of index $i with...
                      nqp::bindpos($!next,$i,nqp::if(
                        nqp::iscont(my $elem := nqp::atpos($iterables,$i))
                          || nqp::not_i(nqp::istype($elem,Iterable)),

                        # single value same as reified list of 1
                        nqp::bindpos(
                          nqp::bindpos($!reifieds,$i,nqp::list),
                          0,
                          $elem
                        ),

                        # something more elaborate
                        nqp::if(
                          nqp::istype($elem,List)
                            && nqp::not_i(nqp::isnull(
                                 nqp::getattr($elem,List,'$!todo'))),

                          # it's a List, may have a reified we can use directly
                          nqp::if(
                            nqp::isnull(
                              $elem := nqp::getattr($elem,List,'$!reified'))
                              || nqp::iseq_i(nqp::elems($elem),0),

                            # cross with an empty list is always an empty list
                            (return Rakudo::Iterator.EmptyIterator),

                            # use the available reified directly
                            nqp::stmts(
                              nqp::bindpos($!reifieds,$i,$elem),
                              nqp::atpos($elem,0)
                            )
                          ),

                          # need to set up an iterator
                          nqp::stmts(
                            nqp::if($elem.is-lazy,($!lazy = 1)),
                            nqp::if(
                              nqp::eqaddr(
                                (my $pulled :=
                                  ($elem := $elem.iterator).pull-one),
                                IterationEnd
                              ),

                              # cross with an empty list is an empty list
                              (return Rakudo::Iterator.EmptyIterator),

                              # set up the iterator stuff
                              nqp::stmts(
                                nqp::bindpos($!iterators,$i,$elem),
                                nqp::bindpos($!reifieds,$i,nqp::list($pulled)),
                                $pulled
                              )
                            )
                          )
                        )
                      ))
                    ),

                    # indices start with 0 xx $elems
                    ($!indices := nqp::setelems(nqp::list_i,$elems)),
                    ($!top = nqp::sub_i($elems,1)),
                    self
                  )
              }
              method new(\iterables) { nqp::create(self)!SET-SELF(iterables) }
              method pull-one() {
                  nqp::if(
                    nqp::isnull($!next),
                    IterationEnd,
                    nqp::stmts(

                      # set up result of this pull
                      (my $result := nqp::p6bindattrinvres(
                        nqp::create(List),List,'$!reified',nqp::clone($!next))),

                      # start working on next result
                      nqp::unless(
                        nqp::isnull(nqp::atpos($!iterators,$!top)),

                        # top level is still iterator, fetch
                        nqp::if(
                          nqp::eqaddr(
                            (my $pulled :=
                              nqp::atpos($!iterators,$!top).pull-one),
                            IterationEnd
                          ),
                          # iterator no more
                          nqp::bindpos($!iterators,$!top,nqp::null),

                          # push value, let normal reifier handler handle
                          nqp::push(
                            nqp::atpos($!reifieds,$!top),
                            $pulled
                          )
                        )
                      ),

                      # no iterator, must use reified list
                      nqp::if(
                        nqp::islt_i(
                          (my int $index =
                            nqp::add_i(nqp::atpos_i($!indices,$!top),1)),
                          nqp::elems(nqp::atpos($!reifieds,$!top))
                        ),

                        # within range, update next result and index
                        nqp::bindpos($!next,$!top,
                          nqp::atpos(
                            nqp::atpos($!reifieds,$!top),
                            nqp::bindpos_i($!indices,$!top,$index)
                          )
                        ),

                        # need to update lower levels
                        nqp::stmts(

                          # update topmost value (go back to first)
                          nqp::bindpos($!next,$!top,
                            nqp::atpos(
                              nqp::atpos($!reifieds,$!top),
                              nqp::bindpos_i($!indices,$!top,0)
                            )
                          ),

                          # until we're at the bottom
                          (my int $level = $!top),
                          nqp::while(
                            nqp::isge_i(($level = nqp::sub_i($level,1)),0),
                            nqp::if(
                              nqp::isnull(nqp::atpos($!iterators,$level)),

                              # can use reified list at this level
                              nqp::if(
                                nqp::islt_i(
                                  ($index = nqp::add_i(
                                    nqp::atpos_i($!indices,$level),1)),
                                  nqp::elems(nqp::atpos($!reifieds,$level))
                                ),

                                # within range, update next result and index
                                nqp::stmts(
                                  nqp::bindpos($!next,$level,
                                    nqp::atpos(
                                      nqp::atpos($!reifieds,$level),
                                      nqp::bindpos_i($!indices,$level,$index)
                                    )
                                  ),
                                  ($level = -1)  # done searching
                                ),

                                # reset this level
                                nqp::bindpos($!next,$level,
                                  nqp::atpos(
                                    nqp::atpos($!reifieds,$level),
                                    nqp::bindpos_i($!indices,$level,0)
                                  )
                                )
                              ),

                              # still an iterator at this level
                              nqp::if(
                                nqp::eqaddr(
                                  ($pulled :=
                                    nqp::atpos($!iterators,$level).pull-one),
                                  IterationEnd
                                ),

                                # exhausted iterator, reset to reified
                                nqp::stmts(
                                  nqp::bindpos($!iterators,$level,nqp::null),
                                  nqp::bindpos($!next,$level,
                                    nqp::atpos(
                                      nqp::atpos($!reifieds,$level),
                                      nqp::bindpos_i($!indices,$level,0)
                                    )
                                  )
                                ),

                                # new value, add to reified, update indices
                                nqp::stmts(
                                  nqp::bindpos(
                                    $!next,
                                    $level,
                                  nqp::bindpos(
                                      nqp::atpos($!reifieds,$level),
                                      nqp::bindpos_i(
                                        $!indices,
                                        $level,
                                        nqp::add_i(
                                          nqp::atpos_i($!indices,$level),
                                          1
                                        )
                                      ),
                                      $pulled
                                    )
                                  ),
                                  ($level = -1)  # done searching
                                )
                            )
                            )
                          ),
                          nqp::if(
                            nqp::iseq_i($level,-1),

                            # was last iteration, free up everything now
                            ($!next :=
                              $!iterators := $!reifieds := $!indices :=
                              nqp::null)
                          )
                        )
                      ),
                      $result
                    )
                  )
              }
              method is-lazy() { nqp::p6bool($!lazy) }
          }.new(@iterables),

          # simpler cases
          nqp::if(
            nqp::iseq_i($n,0),
            # nothing to cross, so return an empty list
            Rakudo::Iterator.Empty,
            # only 1 list to cross, which is the list itself
            nqp::atpos(nqp::getattr(@iterables,List,'$!reified'),0).iterator
          )
        )
    }

    # Return an iterator that will cross the given iterables and map
    # the result with the given mapper Callable.  Basically the
    # functionality of @a Xop @b (with the op  -> mapper functionality
    # to be supplied externally).
    method CrossIterablesMap(@iterables,&mapper) {
        nqp::if(
          nqp::isgt_i((my int $n = @iterables.elems),1),  # reifies

          # actually need to do some crossing (probably)
          class :: does Iterator {
              has $!iterators;  # iterator per iterable, if any
              has $!reifieds;   # cached values (either complete, or so far)
              has $!indices;    # indices of virtual matrix of crossed values
              has $!next;       # IterationBuffer with next values to return
              has $!mapper;     # Callable to do final result mapping
              has int $!lazy;   # whether the outer iterator is lazy
              has int $!top;    # index of top reified/iterator

              method !SET-SELF(\iterables,\mapper) {
                  nqp::stmts(
                    (my $iterables := nqp::getattr(iterables,List,'$!reified')),
                    (my int $elems  = nqp::elems($iterables)),
                    ($!iterators   := nqp::setelems(nqp::list,$elems)),
                    ($!reifieds    := nqp::setelems(nqp::list,$elems)),
                    ($!next :=
                      nqp::setelems(nqp::create(IterationBuffer),$elems)),

                    # loop over all iterables
                    (my int $i = -1),
                    nqp::while(
                      nqp::islt_i(($i = nqp::add_i($i,1)),$elems),

                      # set up initial value of index $i with...
                      nqp::bindpos($!next,$i,nqp::if(
                        nqp::iscont(my $elem := nqp::atpos($iterables,$i))
                          || nqp::not_i(nqp::istype($elem,Iterable)),

                        # single value same as reified list of 1
                        nqp::bindpos(
                          nqp::bindpos($!reifieds,$i,nqp::list),
                          0,
                          $elem
                        ),

                        # something more elaborate
                        nqp::if(
                          nqp::istype($elem,List)
                            && nqp::not_i(nqp::isnull(
{'fraction_non_alphanumeric': 0.11964932618461092, 'fraction_numerical': 0.00343428488624837, 'mean_word_length': 0.9785831015797471, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 140, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6584106', 'n_tokens_mistral': 18521, 'n_tokens_neox': 17614, 'n_words': 4035}