<commit_before>#!/usr/bin/env perl

use 5.010;
use Marpa::R2;
use Data::Dumper;

my $source = <<'SOURCE';
product ::= sku nl name nl price price price nl

sku       ~ sku_0 '.' sku_0
sku_0     ~ [\d]+

price     ~ price_0 ',' price_0
price_0   ~ [\d]+
nl        ~ [\n]

sp        ~ [ ]+
:discard  ~ sp

event '^name' = predicted <name>
:lexeme ~ <name> pause => after event => 'name$'
{ current lexer is 'slurp name' }
name      ~ [^\n]+

SOURCE

my $g = Marpa::R2::Scanless::G->new({
    source => \$source,
    default_action => '::array',
});

my $input = <<'INPUT';
130.12312
Descriptive line
1,10 1,10 1,30
INPUT

my $slr =
    Marpa::R2::Scanless::R->new( { grammar => $g, trace_terminals => 1 } );
my $length = length $input;
READ: for ( my $pos = $slr->read( \$input ); $pos < $length; $pos = $slr->resume() ) {
    EVENT:
    for my $event ( @{ $slr->events() } ) {
        my ($name) = @{$event};
	if ($name eq '^name') {
	   $slr->lexer_set('slurp name');
	   next EVENT;
	}
	if ($name eq 'name$') {
	   $slr->lexer_set('L0');
	   next EVENT;
	}
        die "Unexpected event: ", $name;
    }
}<commit_msg>Work on multiple SLIF lexers: t+<commit_after>#!/usr/bin/env perl

use 5.010;
use Marpa::R2;
use Data::Dumper;

my $source = <<'SOURCE';
product ::= sku nl name nl price price price nl

sku       ~ sku_0 '.' sku_0
sku_0     ~ [\d]+

price     ~ price_0 ',' price_0
price_0   ~ [\d]+
nl        ~ [\n]

sp        ~ [ ]+
:discard  ~ sp

event '^name' = predicted <name>
:lexeme ~ <name> pause => after event => 'name$'
{ current lexer is 'slurp name' }
name      ~ [^\n]+

SOURCE

my $g = Marpa::R2::Scanless::G->new({
    source => \$source,
    default_action => '::array',
});

my $input = <<'INPUT';
130.12312
Descriptive line
1,10 1,10 1,30
INPUT

my $slr =
    Marpa::R2::Scanless::R->new( { grammar => $g, trace_terminals => 99 } );
my $length = length $input;
READ: for ( my $pos = $slr->read( \$input ); $pos < $length; $pos = $slr->resume() ) {
    EVENT:
    for my $event ( @{ $slr->events() } ) {
        my ($event_name) = @{$event};
	if ($event_name eq '^name') {
	   $slr->lexer_set('slurp name');
	   next EVENT;
	}
	if ($event_name eq 'name$') {
	   $slr->lexer_set('L0');
	   next EVENT;
	}
        die "Unexpected event: ", $event_name;
    }
}
{'fraction_non_alphanumeric': 0.2054433713784021, 'fraction_numerical': 0.028533801580333626, 'mean_word_length': 2.8367003367003365, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1255807', 'n_tokens_mistral': 970, 'n_tokens_neox': 926, 'n_words': 241}