<commit_before>#!/usr/bin/env perl

use Marpa::R2;
use Data::Dumper;

my $source = <<'SOURCE';
product ::= sku nl name nl price price price nl

sku       ~ sku_0 '.' sku_0
sku_0     ~ [\d]+

name      ~ [^\n]+

price     ~ price_0 ',' price_0
price_0   ~ [\d]+
nl        ~ [\n]

sp        ~ [ ]+
:discard  ~ sp

SOURCE

my $g = Marpa::R2::Scanless::G->new({
    source => \$source,
    default_action => '::array',
});

my $input = <<'INPUT';
130.12312
Descriptive line
1,10 1,10 1,30
INPUT

my $re = Marpa::R2::Scanless::R->new({ grammar => $g, trace_terminals => 1 }); 
$re->read(\$input);


print Dumper([ ${$re->value} ]); <commit_msg>Multi-lexer example provided by <NAME><commit_after>#!/usr/bin/env perl

use Marpa::R2;
use Data::Dumper;

my $source = <<'SOURCE';
product ::= sku nl name nl price price price nl

sku       ~ sku_0 '.' sku_0
sku_0     ~ [\d]+

price     ~ price_0 ',' price_0
price_0   ~ [\d]+
nl        ~ [\n]

sp        ~ [ ]+
:discard  ~ sp

{ current lexer is 'slurp name' }
name      ~ [^\n]+

SOURCE

my $g = Marpa::R2::Scanless::G->new({
    source => \$source,
    default_action => '::array',
});

my $input = <<'INPUT';
130.12312
Descriptive line
1,10 1,10 1,30
INPUT

my $re = Marpa::R2::Scanless::R->new({ grammar => $g, trace_terminals => 1 }); 
$re->read(\$input);


print Dumper([ ${$re->value} ]); 
{'fraction_non_alphanumeric': 0.20867614061331338, 'fraction_numerical': 0.04038893044128646, 'mean_word_length': 3.018018018018018, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5856212', 'n_tokens_mistral': 580, 'n_tokens_neox': 552, 'n_words': 137}