<commit_before>#!perl
# Copyright 2012 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

# the example grammar in Aycock/Horspool "Practical Earley Parsing",
# _The Computer Journal_, Vol. 45, No. 6, pp. 620-630,
# in its "NNF" form

use 5.010;
use strict;
use warnings;

use English qw( -no_match_vars );
use Test::More tests => 6;
use lib 'inc';
use Marpa::R2::Test;
use Marpa::R2;

my $rules = <<'END_OF_GRAMMAR';
reduce_op ::=
    '+' action => do_arg0
  | '-' action => do_arg0
  | '/' action => do_arg0
  | '*' action => do_arg0
script ::= e action => do_arg0
script ::= script ';' e action => do_arg2
e ::=
     NUM action => do_arg0
   | VAR action => do_is_var
   | '(' e ')' action => do_arg1 assoc => group
  || '-' e action => do_negate
  || e '^' e action => do_power assoc => right
  || e '*' e action => do_multiply
   | e '/' e action => do_divide
  || e '+' e action => do_addition
   | e '-' e action => do_subtract
  || e ',' e action => do_array
  || reduce_op kw_reduce e action => do_reduce
  || VAR op_assign e action => do_set_var
END_OF_GRAMMAR

my $grammar = Marpa::R2::Grammar->new(
    {   start          => 'script',
        actions        => __PACKAGE__,
        rules          => $rules,
    }
);
$grammar->precompute;

# Order matters !!
my @terminals = (
    [ kw_reduce => qr/reduce\b/xms ],
    [ NUM =>       qr/\d+/xms ],
    [ VAR =>       qr/\w+/xms ],
    [ op_assign =>     qr/[=]/xms ],
    [ q{';'},      qr/[;]/xms ],
    [ q{'*'},      qr/[*]/xms ],
    [ q{'/'},      qr/[\/]/xms ],
    [ q{'+'},      qr/[+]/xms ],
    [ q{'-'},      qr/[-]/xms ],
    [ q{'^'},      qr/[\^]/xms ],
    [ q{'('},      qr/[(]/xms ],
    [ q{')'},      qr/[)]/xms ],
    [ q{','},      qr/[,]/xms ],
);

my %symbol_table = ();

sub do_is_var {
    my ( undef, $var ) = @_;
    my $value = $symbol_table{$var};
    die qq{Undefined variable "$var"} if not defined $value;
    return $value;
} ## end sub do_is_var
<commit_msg>Work on Stuifzand if: t+<commit_after>#!perl
# Copyright 2012 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

# the example grammar in Aycock/Horspool "Practical Earley Parsing",
# _The Computer Journal_, Vol. 45, No. 6, pp. 620-630,
# in its "NNF" form

use 5.010;
use strict;
use warnings;

use English qw( -no_match_vars );
use Test::More tests => 6;
use lib 'inc';
use Marpa::R2::Test;
use Marpa::R2;

my $rules = <<'END_OF_GRAMMAR';
reduce_op ::=
    op_plus action => do_arg0
  | op_minus action => do_arg0
  | op_divide action => do_arg0
  | op_star action => do_arg0
script ::= e action => do_arg0
script ::= script op_semicolon e action => do_arg2
e ::=
     NUM action => do_arg0
   | VAR action => do_is_var
   | op_lparen e op_rparen action => do_arg1 assoc => group
  || op_minus e action => do_negate
  || e op_caret e action => do_power assoc => right
  || e op_star e action => do_multiply
   | e op_divide e action => do_divide
  || e op_plus e action => do_addition
   | e op_minus e action => do_subtract
  || e op_comma e action => do_array
  || reduce_op op_reduce e action => do_reduce
  || VAR op_assign e action => do_set_var
END_OF_GRAMMAR

my $grammar = Marpa::R2::Grammar->new(
    {   start          => 'script',
        actions        => __PACKAGE__,
        rules          => $rules,
    }
);
$grammar->precompute;

# Order matters !!
my @terminals = (
    [ op_reduce     => qr/reduce\b/xms ],
    [ NUM           => qr/\d+/xms ],
    [ VAR           => qr/\w+/xms ],
    [ op_assign     => qr/[=]/xms ],
    [ op_semicolon => qr/[;]/xms ],
    [ op_star       => qr/[*]/xms ],
    [ op_divide     => qr/[\/]/xms ],
    [ op_plus       => qr/[+]/xms ],
    [ op_minus      => qr/[-]/xms ],
    [ op_caret      => qr/[\^]/xms ],
    [ op_lparen     => qr/[(]/xms ],
    [ op_rparen     => qr/[)]/xms ],
    [ op_comma      => qr/[,]/xms ],
);

my %symbol_table = ();

sub do_is_var {
    my ( undef, $var ) = @_;
    my $value = $symbol_table{$var};
    die qq{Undefined variable "$var"} if not defined $value;
    return $value;
} ## end sub do_is_var

{'fraction_non_alphanumeric': 0.1539909638554217, 'fraction_numerical': 0.012801204819277108, 'mean_word_length': 2.7336612789880532, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6734314', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1879, 'n_words': 592}