<commit_before>#!perl
# Copyright 2012 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

use 5.010;
use warnings;
use strict;

use English qw( -no_match_vars );
use PPI 1.206;
use Marpa::R2;
use lib 'pperl';
use Marpa::R2::Perl;

my %hash;
my %codeblock;

my @tests;
my $string = do { local $RS = undef; <STDIN> };

my $finder = Marpa::R2::Perl->new( { embedded => 1, closures => {} } );
my $parser = Marpa::R2::Perl->new( { closures => {} } );

$parser = $parser->read( \$string );
my @values    = $parser->eval();
my $recce     = $parser->{recce};
my $grammar   = $recce->[Marpa::R2::Internal::Recognizer::GRAMMAR];
my $grammar_c = $grammar->[Marpa::R2::Internal::Grammar::C];
my $rules     = $grammar->[Marpa::R2::Internal::Grammar::RULES];
for my $earley_set_id ( 0 .. $recce->latest_earley_set() ) {
    my $progress_report = $recce->progress($earley_set_id);
    ITEM: for my $progress_item ( @{$progress_report} ) {
        my ( $rule_id, $position, $origin_earley_set_id ) = @{$progress_item};
        last ITEM if not defined $rule_id;
        next ITEM if $position >= 0;
        $position = $grammar_c->rule_length($rule_id);

        my $origin_earleme = $recce->earleme($origin_earley_set_id);

        my $rule      = $rules->[$rule_id];
        my $rule_name = $rule->[Marpa::R2::Internal::Rule::NAME];
        next ITEM if not defined $rule_name;
        my $blocktype =
              $rule_name eq 'anon_hash' ? 'hash'
            : $rule_name eq 'block'     ? 'code'
            : $rule_name eq 'mblock'    ? 'code'
            :                             undef;<commit_msg>Work on Perl parser: t+<commit_after>#!perl
# Copyright 2012 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

use 5.010;
use warnings;
use strict;

use English qw( -no_match_vars );
use PPI 1.206;
use Marpa::R2;
use lib 'pperl';
use Marpa::R2::Perl;

my %hash;
my %codeblock;

my @tests;
my $string = do { local $RS = undef; <STDIN> };

my $finder = Marpa::R2::Perl->new( { embedded => 1, closures => {} } );
my $parser = Marpa::R2::Perl->new( { closures => {} } );

my $tokens = $finder->tokens(\$string);
my $start = 0;
my $last_end = 0;
PERL_CODE: while (1) {
  say STDERR '=' x 40;
  say STDERR "last_end=$last_end";
  my ($start, $end) = $finder->find_perl( $last_end );
  last PERL_CODE if not defined $start;
  say STDERR join q{ }, ($start // '-'), ($end // '-');
  say STDERR $finder->{PPI_tokens}->[$end]->content();
  say STDERR map { $_->content() } @{$tokens}[($last_end//0) .. $end];
  $last_end = $end;
}

exit 0;

my $recce     = $parser->{recce};
my $grammar   = $recce->[Marpa::R2::Internal::Recognizer::GRAMMAR];
my $grammar_c = $grammar->[Marpa::R2::Internal::Grammar::C];
my $rules     = $grammar->[Marpa::R2::Internal::Grammar::RULES];
for my $earley_set_id ( 0 .. $recce->latest_earley_set() ) {
    my $progress_report = $recce->progress($earley_set_id);
    ITEM: for my $progress_item ( @{$progress_report} ) {
        my ( $rule_id, $position, $origin_earley_set_id ) = @{$progress_item};
        last ITEM if not defined $rule_id;
        next ITEM if $position >= 0;
        $position = $grammar_c->rule_length($rule_id);

        my $origin_earleme = $recce->earleme($origin_earley_set_id);

        my $rule      = $rules->[$rule_id];
        my $rule_name = $rule->[Marpa::R2::Internal::Rule::NAME];
        next ITEM if not defined $rule_name;
        my $blocktype =
              $rule_name eq 'anon_hash' ? 'hash'
            : $rule_name eq 'block'     ? 'code'
            : $rule_name eq 'mblock'    ? 'code'
            :                             undef;
{'fraction_non_alphanumeric': 0.1488180883864337, 'fraction_numerical': 0.012949640287769784, 'mean_word_length': 3.3100088573959257, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5651827', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1644, 'n_words': 508}