<commit_before>my class IO::Handle does IO::FileTestable {
    has $!PIO;
    has int $.ins;
    has $.chomp = Bool::True;
    has $.path;
    has Bool $!isDir;

    proto method open(|) { * }
    multi method open($path? is copy, :$r is copy, :$w is copy, :$rw, :$a, :$p, :$bin, :$chomp = Bool::True,
            :enc(:$encoding) = 'utf8') {
        $path //= $!path;
        my $is_std_handle = $path eq "-";
        $r = $w = True if $rw;
        my $abspath = !$is_std_handle && defined($*CWD) ?? IO::Spec.rel2abs($path) !! $path;
        $!isDir = Bool::True if !$is_std_handle &&
            nqp::p6bool(nqp::stat($abspath.Str, nqp::const::STAT_EXISTS))
            && nqp::p6bool(nqp::stat($abspath.Str, nqp::const::STAT_ISDIR));
        fail (X::IO::Directory.new(:$path, :trying<open( :w )>))
            if $w && $!isDir;
#?if parrot
        my $mode =  $p ?? ($w ||  $a ?? 'wp' !! 'rp') !!
                   ($w ?? 'w' !! ($a ?? 'wa' !! 'r' ));
        # TODO: catch error, and fail()
        nqp::bindattr(self, IO::Handle, '$!PIO',
             $is_std_handle
                ?? ( $w || $a ?? nqp::getstdout() !! nqp::getstdin() )
                !! nqp::open(nqp::unbox_s($abspath.Str), nqp::unbox_s($mode))
        );
#?endif
#?if !parrot
        if $p {
            #~ my $mode =  $p ?? ($w ||  $a ?? 'wp' !! 'rp');

            my Mu $hash-with-containers := nqp::getattr(%*ENV, EnumMap, '$!storage');
            my Mu $hash-without         := nqp::hash();
            my Mu $enviter := nqp::iterator($hash-with-containers);
            my $envelem;
            while $enviter {
                $envelem := nqp::shift($enviter);
                nqp::bindkey($hash-without, nqp::iterkey_s($envelem), nqp::decont(nqp::iterval($envelem)))
            }

            my $errpath = '';
            nqp::bindattr(self, IO::Handle, '$!PIO',
                 $is_std_handle
                    ?? ( $w || $a ?? nqp::getstdout() !! nqp::getstdin() )
                    !! nqp::openpipe(nqp::unbox_s($abspath.Str), nqp::unbox_s($*CWD.Str), $hash-without, nqp::unbox_s($errpath))
            );
        }
        else {
            my $mode =  $w ?? 'w' !! ($a ?? 'wa' !! 'r' );
            # TODO: catch error, and fail()
            nqp::bindattr(self, IO::Handle, '$!PIO',
                 $is_std_handle
                    ?? ( $w || $a ?? nqp::getstdout() !! nqp::getstdin() )
                    !! nqp::open(nqp::unbox_s($abspath.Str), nqp::unbox_s($mode))
            );
        }
#?endif
        $!path = $path;
        $!chomp = $chomp;
        nqp::setencoding($!PIO, NORMALIZE_ENCODING($encoding)) unless $bin;
        self;
    }

    method close() {
        # TODO:b catch errors
        nqp::closefh($!PIO) if nqp::defined($!PIO);
        $!PIO := Mu;
        Bool::True;
    }

    method eof() {
        nqp::p6bool(nqp::eoffh($!PIO));
    }

    method get() {
        unless nqp::defined($!PIO) {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<get>)) if $!isDir;
        return Str if self.eof;
        my Str $x = nqp::p6box_s(nqp::readlinefh($!PIO));
        # XXX don't fail() as long as it's fatal
        # fail('end of file') if self.eof && $x eq '';
        $x.=chomp if $.chomp;
        return Str if self.eof && $x eq '';

        $!ins = $!ins + 1;
        $x;
    }
    
    method getc() {
        unless $!PIO {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<getc>)) if $!isDir;
        my $c = nqp::p6box_s(nqp::getcfh($!PIO));
        fail if $c eq '';
        $c;
    }

    proto method lines (|) { * }
    multi method lines(IO::Handle:D:) {<commit_msg>Implement IO::Handle.words<commit_after>my class IO::Handle does IO::FileTestable {
    has $!PIO;
    has int $.ins;
    has $.chomp = Bool::True;
    has $.path;
    has Bool $!isDir;

    proto method open(|) { * }
    multi method open($path? is copy, :$r is copy, :$w is copy, :$rw, :$a, :$p, :$bin, :$chomp = Bool::True,
            :enc(:$encoding) = 'utf8') {
        $path //= $!path;
        my $is_std_handle = $path eq "-";
        $r = $w = True if $rw;
        my $abspath = !$is_std_handle && defined($*CWD) ?? IO::Spec.rel2abs($path) !! $path;
        $!isDir = Bool::True if !$is_std_handle &&
            nqp::p6bool(nqp::stat($abspath.Str, nqp::const::STAT_EXISTS))
            && nqp::p6bool(nqp::stat($abspath.Str, nqp::const::STAT_ISDIR));
        fail (X::IO::Directory.new(:$path, :trying<open( :w )>))
            if $w && $!isDir;
#?if parrot
        my $mode =  $p ?? ($w ||  $a ?? 'wp' !! 'rp') !!
                   ($w ?? 'w' !! ($a ?? 'wa' !! 'r' ));
        # TODO: catch error, and fail()
        nqp::bindattr(self, IO::Handle, '$!PIO',
             $is_std_handle
                ?? ( $w || $a ?? nqp::getstdout() !! nqp::getstdin() )
                !! nqp::open(nqp::unbox_s($abspath.Str), nqp::unbox_s($mode))
        );
#?endif
#?if !parrot
        if $p {
            #~ my $mode =  $p ?? ($w ||  $a ?? 'wp' !! 'rp');

            my Mu $hash-with-containers := nqp::getattr(%*ENV, EnumMap, '$!storage');
            my Mu $hash-without         := nqp::hash();
            my Mu $enviter := nqp::iterator($hash-with-containers);
            my $envelem;
            while $enviter {
                $envelem := nqp::shift($enviter);
                nqp::bindkey($hash-without, nqp::iterkey_s($envelem), nqp::decont(nqp::iterval($envelem)))
            }

            my $errpath = '';
            nqp::bindattr(self, IO::Handle, '$!PIO',
                 $is_std_handle
                    ?? ( $w || $a ?? nqp::getstdout() !! nqp::getstdin() )
                    !! nqp::openpipe(nqp::unbox_s($abspath.Str), nqp::unbox_s($*CWD.Str), $hash-without, nqp::unbox_s($errpath))
            );
        }
        else {
            my $mode =  $w ?? 'w' !! ($a ?? 'wa' !! 'r' );
            # TODO: catch error, and fail()
            nqp::bindattr(self, IO::Handle, '$!PIO',
                 $is_std_handle
                    ?? ( $w || $a ?? nqp::getstdout() !! nqp::getstdin() )
                    !! nqp::open(nqp::unbox_s($abspath.Str), nqp::unbox_s($mode))
            );
        }
#?endif
        $!path = $path;
        $!chomp = $chomp;
        nqp::setencoding($!PIO, NORMALIZE_ENCODING($encoding)) unless $bin;
        self;
    }

    method close() {
        # TODO:b catch errors
        nqp::closefh($!PIO) if nqp::defined($!PIO);
        $!PIO := Mu;
        Bool::True;
    }

    method eof() {
        nqp::p6bool(nqp::eoffh($!PIO));
    }

    method get() {
        unless nqp::defined($!PIO) {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<get>)) if $!isDir;
        return Str if self.eof;
        my Str $x = nqp::p6box_s(nqp::readlinefh($!PIO));
        # XXX don't fail() as long as it's fatal
        # fail('end of file') if self.eof && $x eq '';
        $x.=chomp if $.chomp;
        return Str if self.eof && $x eq '';

        $!ins = $!ins + 1;
        $x;
    }
    
    method getc() {
        unless $!PIO {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<getc>)) if $!isDir;
        my $c = nqp::p6box_s(nqp::getcfh($!PIO));
        fail if $c eq '';
        $c;
    }

    proto method words (|) { * }
    multi method words(IO::Handle:D:) {
        unless nqp::defined($!PIO) {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<words>)) if $!isDir;

        my str $str;
        my int $chars;
        my int $pos;
        my int $left;
        my int $nextpos;

        gather until nqp::eoffh($!PIO) {

            my Buf $buf := Buf.new;
            nqp::readfh($!PIO, $buf, 65536);

            $str = $str ~ nqp::unbox_s($buf.decode);
            $chars = nqp::chars($str);
            $pos   = nqp::findnotcclass(
              nqp::const::CCLASS_WHITESPACE, $str, 0, $chars);

            while ($left = $chars - $pos) > 0 {
                $nextpos = nqp::findcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $pos, $left);
                last unless $left = $chars - $nextpos; # broken word

                take
                  nqp::box_s(nqp::substr($str, $pos, $nextpos - $pos), Str);

                $pos = nqp::findnotcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $nextpos, $left);
            }

            $str = nqp::substr($str,$pos) if $pos < $chars;
        }
    }
    multi method words(IO::Handle:D: :$eager!) { # can probably go after GLR
        return self.words if !$eager;

        unless nqp::defined($!PIO) {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<words>)) if $!isDir;

        my str $str;
        my int $chars;
        my int $pos;
        my int $left;
        my int $nextpos;
        my Mu $rpa := nqp::list();

        until nqp::eoffh($!PIO) {

            my Buf $buf := Buf.new;
            nqp::readfh($!PIO, $buf, 65536);

            $str   = $str ~ nqp::unbox_s($buf.decode);
            $chars = nqp::chars($str);
            $pos   = nqp::findnotcclass(
              nqp::const::CCLASS_WHITESPACE, $str, 0, $chars);

            while ($left = $chars - $pos) > 0 {
                $nextpos = nqp::findcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $pos, $left);
                last unless $left = $chars - $nextpos; # broken word

                nqp::push($rpa,
                  nqp::box_s(nqp::substr($str, $pos, $nextpos - $pos), Str) );

                $pos = nqp::findnotcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $nextpos, $left);
            }

            $str = nqp::substr($str,$pos) if $pos < $chars;
        }
        nqp::p6parcel($rpa, Nil);
    }
    multi method words(IO::Handle:D: :$count!) {
        return self.words if !$count;

        unless nqp::defined($!PIO) {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<words>)) if $!isDir;

        my str $str;
        my int $chars;
        my int $pos;
        my int $left;
        my int $nextpos;
        my int $found;

        until nqp::eoffh($!PIO) {

            my Buf $buf := Buf.new;
            nqp::readfh($!PIO, $buf, 65536);

            $str   = $str ~ nqp::unbox_s($buf.decode);
            $chars = nqp::chars($str);
            $pos   = nqp::findnotcclass(
              nqp::const::CCLASS_WHITESPACE, $str, 0, $chars);

            while ($left = $chars - $pos) > 0 {
                $nextpos = nqp::findcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $pos, $left);
                last unless $left = $chars - $nextpos; # broken word

                $found = $found + 1;

                $pos = nqp::findnotcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $nextpos, $left);
            }

            $str = nqp::substr($str,$pos) if $pos < $chars;
        }
        nqp::box_i($found, Int);
    }
    multi method words(IO::Handle:D: $limit, :$eager) {
        return self.words(:$eager) if $limit == Inf or $limit ~~ Whatever;

        unless nqp::defined($!PIO) {
            self.open($!path, :chomp($.chomp));
        }
        fail (X::IO::Directory.new(:$!path, :trying<words>)) if $!isDir;

        my str $str;
        my int $chars;
        my int $pos;
        my int $left;
        my int $nextpos;
        my int $count = $limit;
        my Mu $rpa := nqp::list();

        until nqp::eoffh($!PIO) {

            my Buf $buf := Buf.new;
            nqp::readfh($!PIO, $buf, 65536);

            $str   = $str ~ nqp::unbox_s($buf.decode);
            $chars = nqp::chars($str);
            $pos   = nqp::findnotcclass(
              nqp::const::CCLASS_WHITESPACE, $str, 0, $chars);

            while $count and ($left = $chars - $pos) > 0 {
                $nextpos = nqp::findcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $pos, $left);
                last unless $left = $chars - $nextpos; # broken word

                nqp::push($rpa,
                  nqp::box_s(nqp::substr($str, $pos, $nextpos - $pos), Str) );
                $count = $count - 1;

                $pos = nqp::findnotcclass(
                  nqp::const::CCLASS_WHITESPACE, $str, $nextpos, $left);
            }

            $str = nqp::substr($str,$pos) if $pos < $chars;
        }
        nqp::p6parcel($rpa, Nil);
    }

    proto method lines (|) { * }
    multi method lines(IO::Handle:D:) {
{'fraction_non_alphanumeric': 0.19058526182765975, 'fraction_numerical': 0.0037911697338282916, 'mean_word_length': 1.78837260515305, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2092435', 'n_tokens_mistral': 4619, 'n_tokens_neox': 4490, 'n_words': 919}