<commit_before>#!/usr/bin/perl
# Copyright 2013 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

use 5.010;
use strict;
use warnings;
use English qw( -no_match_vars );
use Data::Dumper;
use Scalar::Util qw(blessed);

# This is a 'meta' tool, so I relax some of the
# restrictions I use to guarantee portability.
use autodie;

# I expect to be run from a subdirectory in the
# development heirarchy
use lib '../../../';
use lib '../../../../blib/arch';
use Marpa::R2;
BEGIN { require './Try.pm'; }

use Getopt::Long;
my $verbose          = 1;
my $help_flag        = 0;
my $scannerless_flag = 1;
my $result           = Getopt::Long::GetOptions(
    'help'      => \$help_flag,
);
die "usage $PROGRAM_NAME [--help] file ...\n" if $help_flag;

my $bnf           = do { local $RS = undef; \(<>) };
my $parse_result =
    Marpa::R2::Scanless::G->_source_to_ast( $bnf );
my $self = {};
say "ast result = \n", Data::Dumper::Dumper(doit($self, $parse_result));
say "self object = \n", Data::Dumper::Dumper($self);

exit 0;

sub doit {
    my ($self, $value) = @_;
    my $ref_type = ref $value;
    return $value->doit($self) if Scalar::Util::blessed($value);
    return [ map { doit($self, $_) } @{$value} ] if $ref_type eq 'ARRAY';
    return $value;
} ## end sub doit

sub sort_bnf {
    my $cmp = $a->{lhs} cmp $b->{lhs};
    return $cmp if $cmp;
    my $a_rhs_length = scalar @{ $a->{rhs} };
    my $b_rhs_length = scalar @{ $b->{rhs} };
    $cmp = $a_rhs_length <=> $b_rhs_length;
    return $cmp if $cmp;
    for my $ix ( 0 .. ($a_rhs_length-1) ) {
        $cmp = $a->{rhs}->[$ix] cmp $b->{rhs}->[$ix];
        return $cmp if $cmp;
    }
    return 0;
} ## end sub sort_bnf

my %cooked_parse_result = (
    is_lexeme         => $parse_result->{is_lexeme},
    character_classes => $parse_result->{character_classes}
);
for my $rule_set (qw(lex_rules g1_rules)) {
    my $aoh        = $parse_result->{$rule_set};
    my $sorted_aoh = [ sort sort_bnf @{$aoh} ];
    $cooked_parse_result{$rule_set} = $sorted_aoh;
}

say "## The code after this line was automatically generated by ",
    $PROGRAM_NAME;
say "## Date: ", scalar localtime();
$Data::Dumper::Sortkeys = 1;
print Data::Dumper->Dump( [ \%cooked_parse_result ], [qw(hashed_metag)] );
say "## The code before this line was automatically generated by ",
    $PROGRAM_NAME;<commit_msg>Work on AST: t+<commit_after>#!/usr/bin/perl
# Copyright 2013 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

use 5.010;
use strict;
use warnings;
use English qw( -no_match_vars );
use Data::Dumper;
use Carp;
use Scalar::Util qw(blessed reftype);

# This is a 'meta' tool, so I relax some of the
# restrictions I use to guarantee portability.
use autodie;

# I expect to be run from a subdirectory in the
# development heirarchy
use lib '../../../';
use lib '../../../../blib/arch';
use Marpa::R2;
BEGIN { require './Try.pm'; }

use Getopt::Long;
my $verbose          = 1;
my $help_flag        = 0;
my $scannerless_flag = 1;
my $result           = Getopt::Long::GetOptions(
    'help'      => \$help_flag,
);
die "usage $PROGRAM_NAME [--help] file ...\n" if $help_flag;

my $bnf           = do { local $RS = undef; \(<>) };
my $parse_result =
    Marpa::R2::Scanless::G->_source_to_ast( $bnf );
die "_source_to_ast did not return an AST" if not ref $parse_result eq 'REF';
my $self = {};
say "ast result = \n", Data::Dumper::Dumper(evaluate($self, ${$parse_result}));
say "self object = \n", Data::Dumper::Dumper($self);

exit 0;

sub evaluate {
    my ($self, $value) = @_;
    return $value if not defined $value;
    if (Scalar::Util::blessed($value)) {
	return $value->evaluate($self) if $value->can('evaluate');
	return bless [ map { evaluate($self, $_) } @{$value} ], ref $value if Scalar::Util::reftype($value) eq 'ARRAY';
	return $value;
    }
    return [ map { evaluate($self, $_) } @{$value} ] if ref $value eq 'ARRAY';
    return $value;
} ## end sub evaluate

sub sort_bnf {
    my $cmp = $a->{lhs} cmp $b->{lhs};
    return $cmp if $cmp;
    my $a_rhs_length = scalar @{ $a->{rhs} };
    my $b_rhs_length = scalar @{ $b->{rhs} };
    $cmp = $a_rhs_length <=> $b_rhs_length;
    return $cmp if $cmp;
    for my $ix ( 0 .. ($a_rhs_length-1) ) {
        $cmp = $a->{rhs}->[$ix] cmp $b->{rhs}->[$ix];
        return $cmp if $cmp;
    }
    return 0;
} ## end sub sort_bnf

my %cooked_parse_result = (
    is_lexeme         => $parse_result->{is_lexeme},
    character_classes => $parse_result->{character_classes}
);
for my $rule_set (qw(lex_rules g1_rules)) {
    my $aoh        = $parse_result->{$rule_set};
    my $sorted_aoh = [ sort sort_bnf @{$aoh} ];
    $cooked_parse_result{$rule_set} = $sorted_aoh;
}

say "## The code after this line was automatically generated by ",
    $PROGRAM_NAME;
say "## Date: ", scalar localtime();
$Data::Dumper::Sortkeys = 1;
print Data::Dumper->Dump( [ \%cooked_parse_result ], [qw(hashed_metag)] );
say "## The code before this line was automatically generated by ",
    $PROGRAM_NAME;

exit 0;

package Marpa::R2::Internal::MetaG::Symbol;

# Make the child argument into a symbol, if it is
# not one already
sub evaluate { return $_[0] };
sub new {
    my ( $class, $self ) = @_;
    return bless { name => '' . $self }, $class if ref $self eq q{};
    return $self;
}

package Marpa::R2::Internal::MetaG_Nodes::single_symbol;
sub evaluate { return $_[0]->[0]->evaluate(); }
package Marpa::R2::Internal::MetaG_Nodes::symbol;
sub evaluate { return $_[0]->[0]->evaluate() ; }
package Marpa::R2::Internal::MetaG_Nodes::symbol_name;
sub evaluate { return $_[0]->[0]->evaluate(); }

package Marpa::R2::Internal::MetaG_Nodes::character_class;
sub evaluate { return Marpa::R2::Internal::MetaG::Symbol->new( $_[0]->[0] ); }

package Marpa::R2::Internal::MetaG_Nodes::bare_name;
sub evaluate { return Marpa::R2::Internal::MetaG::Symbol->new( $_[0]->[0] ); }

package Marpa::R2::Internal::MetaG_Nodes::bracketed_name;

sub evaluate {
    my ($children) = @_;
    my $bracketed_name = $children->[0];

    # normalize whitespace
    $bracketed_name =~ s/\A [<] \s*//xms;
    $bracketed_name =~ s/ \s* [>] \z//xms;
    $bracketed_name =~ s/ \s+ / /gxms;
    return Marpa::R2::Internal::MetaG::Symbol->new($bracketed_name);
} ## end sub evaluate

{'fraction_non_alphanumeric': 0.16297679381168312, 'fraction_numerical': 0.009469191784475861, 'mean_word_length': 3.637600494743352, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2983783', 'n_tokens_mistral': 2665, 'n_tokens_neox': 2615, 'n_words': 807}