<commit_before>use v6;
use lib 't/spec/packages';

use Test;
use Test::Tap;

my @simple  = <a b c d e>;
my @original;
my @endings = "\n", "\r", "\r\n";

plan 21;

dies-ok { Supply.lines }, 'can not be called as a class method';

for ThreadPoolScheduler.new, CurrentThreadScheduler -> $*SCHEDULER {
    diag "**** scheduling with {$*SCHEDULER.WHAT.perl}";

    tap-ok Supply.from-list( @simple.map: * ~ "\n" ).lines,
      @simple,
      "handle a simple list of lines with LF";
    tap-ok Supply.from-list( @original = @simple.map: * ~ "\n" ).lines(:!chomp),
      @original,
      "handle a simple list of lines with LF without chomping";

    tap-ok Supply.from-list( @simple.map: * ~ "\r" ).lines,
      @simple,
      "handle a simple list of lines with CR";
    tap-ok Supply.from-list( @original = @simple.map: * ~ "\r" ).lines(:!chomp),
      @original,
      "handle a simple list of lines with CR without chomping";

    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list( @simple.map: * ~ "\r\n" ).lines,
      @simple,
      "handle a simple list of lines with CRLF";
    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list( @original = @simple.map: * ~ "\r\n" ).lines(:!chomp),
      @original,
      "handle a simple list of lines with CRLF without chomping";

    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list( @simple.map: * ~ @endings.pick ).lines,
      @simple,
      "handle a simple list of lines with mixed line ending";
    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list(@original= @simple.map: * ~ @endings.pick).lines(:!chomp),
      @original,
      "handle a simple list of lines with mixed line ending w/o chomping";

    {
        my $s = Supply.new;
        #?rakudo.jvm todo '\r\n not yet handled as grapheme'
        tap-ok $s.lines,
          ['a','b','c','d', '', 'eeee'],
          "handle chunked lines",
          :after-tap( {
              $s.emit( "a\nb\r" );
              $s.emit( "\nc\rd\n" );
              $s.emit( "\ne" );
              $s.emit( "eee" );
              $s.done;
          } );
    }

    {
        my $s = Supply.new;
        #?rakudo.jvm todo '\r\n not yet handled as grapheme'
        tap-ok $s.lines(:!chomp),
          ["a\n","b\r\n","c\r","d\n","\n","eeee"],
          "handle chunked lines",
          :after-tap( {
              $s.emit( "a\nb\r" );
              $s.emit( "\nc\rd\n" );
              $s.emit( "\ne" );
              $s.emit( "eee" );
              $s.done;
          } );
    }
}<commit_msg>Add specific test for chunked CRLF.  See 97b93c3.<commit_after>use v6;
use lib 't/spec/packages';

use Test;
use Test::Tap;

my @simple  = <a b c d e>;
my @original;
my @endings = "\n", "\r", "\r\n";

plan 23;

dies-ok { Supply.lines }, 'can not be called as a class method';

for ThreadPoolScheduler.new, CurrentThreadScheduler -> $*SCHEDULER {
    diag "**** scheduling with {$*SCHEDULER.WHAT.perl}";

    tap-ok Supply.from-list( @simple.map: * ~ "\n" ).lines,
      @simple,
      "handle a simple list of lines with LF";
    tap-ok Supply.from-list( @original = @simple.map: * ~ "\n" ).lines(:!chomp),
      @original,
      "handle a simple list of lines with LF without chomping";

    tap-ok Supply.from-list( @simple.map: * ~ "\r" ).lines,
      @simple,
      "handle a simple list of lines with CR";
    tap-ok Supply.from-list( @original = @simple.map: * ~ "\r" ).lines(:!chomp),
      @original,
      "handle a simple list of lines with CR without chomping";

    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list( @simple.map: * ~ "\r\n" ).lines,
      @simple,
      "handle a simple list of lines with CRLF";
    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list( @original = @simple.map: * ~ "\r\n" ).lines(:!chomp),
      @original,
      "handle a simple list of lines with CRLF without chomping";

    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list( @simple.map: * ~ @endings.pick ).lines,
      @simple,
      "handle a simple list of lines with mixed line ending";
    #?rakudo.jvm todo '\r\n not yet handled as grapheme'
    tap-ok Supply.from-list(@original= @simple.map: * ~ @endings.pick).lines(:!chomp),
      @original,
      "handle a simple list of lines with mixed line ending w/o chomping";

    {
        my $s = Supply.new;
        #?rakudo.jvm todo '\r\n not yet handled as grapheme'
        tap-ok $s.lines,
          ['a','b','c','d', '', 'eeee'],
          "handle chunked lines",
          :after-tap( {
              $s.emit( "a\nb\r" );
              $s.emit( "\nc\rd\n" );
              $s.emit( "\ne" );
              $s.emit( "eee" );
              $s.done;
          } );
    }

    {
        my $s = Supply.new;
        #?rakudo.jvm todo '\r\n not yet handled as grapheme'
        tap-ok $s.lines(:!chomp),
          ["a\n","b\r\n","c\r","d\n","\n","eeee"],
          "handle chunked lines",
          :after-tap( {
              $s.emit( "a\nb\r" );
              $s.emit( "\nc\rd\n" );
              $s.emit( "\ne" );
              $s.emit( "eee" );
              $s.done;
          } );
    }

    # See 97b93c3.
    {
        my $s = Supply.new;
        #?rakudo.jvm todo '\r\n not yet handled as grapheme'
        my $l = $s.lines(:!chomp);
        my @res;
        $l.tap({ @res.push: $_ });
        $s.emit("a\r");
        $s.emit("\n");
        for ^50 { last if @res; sleep .1 }
        is-deeply(@res, ["a\r\n"], "handle chunked line endings");
        $s.done;
    }
}
{'fraction_non_alphanumeric': 0.1761464628510487, 'fraction_numerical': 0.0033771773906861, 'mean_word_length': 1.970960929250264, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7175765', 'n_tokens_mistral': 2081, 'n_tokens_neox': 1990, 'n_words': 555}