<commit_before># Copyright 2013 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

package Marpa::R2::Scanless::G;

use 5.010;
use strict;
use warnings;

use vars qw($VERSION $STRING_VERSION);
$VERSION        = '2.077_014';
$STRING_VERSION = $VERSION;
## no critic(BuiltinFunctions::ProhibitStringyEval)
$VERSION = eval $VERSION;
## use critic

package Marpa::R2::Internal::Scanless::G;

use Scalar::Util 'blessed';
use English qw( -no_match_vars );

# names of packages for strings
our $PACKAGE = 'Marpa::R2::Scanless::G';

sub Marpa::R2::Internal::Scanless::meta_grammar {

    my $meta_slg = bless [], 'Marpa::R2::Scanless::G';
    state $hashed_metag = Marpa::R2::Internal::MetaG::hashed_grammar();
    Marpa::R2::Internal::Scanless::G::hash_to_runtime( $meta_slg,
        $hashed_metag,
        { bless_package => 'Marpa::R2::Internal::MetaAST_Nodes' } );

    my $thick_g1_grammar =
        $meta_slg->[Marpa::R2::Internal::Scanless::G::THICK_G1_GRAMMAR];
    my @mask_by_rule_id;
    $mask_by_rule_id[$_] = $thick_g1_grammar->_rule_mask($_)
        for $thick_g1_grammar->rule_ids();
    $meta_slg->[Marpa::R2::Internal::Scanless::G::MASK_BY_RULE_ID] =
        \@mask_by_rule_id;

    return $meta_slg;

} ## end sub Marpa::R2::Internal::Scanless::meta_grammar

sub Marpa::R2::Scanless::G::new {
    my ( $class, @hash_ref_args ) = @_;

    my $slg = [];
    bless $slg, $class;

    my ($dsl, $g1_args) = Marpa::R2::Internal::Scanless::G::set ( $slg, 'new', @hash_ref_args );
    my $ast = Marpa::R2::Internal::MetaAST->new( $dsl );
    my $hashed_ast = $ast->ast_to_hash();
    Marpa::R2::Internal::Scanless::G::hash_to_runtime($slg, $hashed_ast, $g1_args);
    return $slg;
} ## end sub Marpa::R2::Scanless::G::new

sub Marpa::R2::Scanless::G::set {
    my ( $slg, @hash_ref_args ) = @_;
    Marpa::R2::Internal::Scanless::G::set ( $slg, 'set', @hash_ref_args );
    return $slg;
}

# The context flag indicates whether this set is called directly by the user
# or is for series reset or the constructor.  "Context" flags of this kind
# are much decried practice, and for good reason, but in this case
# I think it is justified.
# This logic really needs to be all in one place, and so a flag
# to trigger the minor differences needed by the various calling
# contexts is a small price to pay.
sub Marpa::R2::Internal::Scanless::G::set {
    my ( $slg, $method, @hash_ref_args ) = @_;

    # Other possible grammar options:
    # actions
    # default_rank
    # inaccessible_ok
    # symbols
    # terminals
    # unproductive_ok
    # warnings

    state $copy_to_g1_args =
        { map { ( $_, 1 ); }
            qw(trace_file_handle action_object default_action bless_package) };
    state $set_method_args =
        { map { ( $_, 1 ); } qw(trace_file_handle) };
    state $internal_args =
        { map { ( $_, 1 ); } qw(bless_package) };
    state $new_method_args = {
        map { ( $_, 1 ); } qw(source),
        keys %{$copy_to_g1_args},
        keys %{$set_method_args}
    };
    for my $args (@hash_ref_args) {
        my $ref_type = ref $args;
        if ( not $ref_type ) {
            Marpa::R2::exception( q{$slr->}
                    . $method
                    . qq{() expects args as ref to HASH; got non-reference instead}
            );
        } ## end if ( not $ref_type )
        if ( $ref_type ne 'HASH' ) {
            Marpa::R2::exception( q{$slr->}
                    . $method
                    . qq{() expects args as ref to HASH, got ref to $ref_type instead}
            );
        } ## end if ( $ref_type ne 'HASH' )
    } ## end for my $args (@hash_ref_args)

    my %flat_args = ();
    for my $hash_ref (@hash_ref_args) {
        ARG: for my $arg_name ( keys %{$hash_ref} ) {
            $flat_args{$arg_name} = $hash_ref->{$arg_name};
        }
    }

    my $ok_args = $set_method_args;
    $ok_args = $new_method_args if $method eq 'new';
    $ok_args = $internal_args if $method eq 'meta';
    my @bad_args = grep { not $ok_args->{$_} } keys %flat_args;
    if ( scalar @bad_args ) {
        Marpa::R2::exception(
            q{Bad named argument(s) to $slr->}
                . $method
                . q{() method: }
                . join q{ },
            @bad_args
        );
    } ## end if ( scalar @bad_args )

    my $dsl;
    if ( $method eq 'new' ) {
        state $arg_name = 'source';
        $dsl = $flat_args{$arg_name};
        Marpa::R2::exception(
            qq{Marpa::R2::Scanless::G::new() called without a "$arg_name" argument}
        ) if not defined $dsl;
        my $ref_type = ref $dsl;
        if ( $ref_type ne 'SCALAR' ) {
            my $desc = $ref_type ? "a ref to $ref_type" : 'not a ref';
            Marpa::R2::exception(
                qq{'$arg_name' name argument to Marpa::R2::Scanless::G->new() is $desc\n},
                "  It should be a ref to a string\n"
            );
        } ## end if ( $ref_type ne 'SCALAR' )
    } ## end if ( $method eq 'new' )

    # A bit hack-ish, but some named args will be copies straight to a member of
    # the Scanless::G class, so this maps named args to the index of the array
    # that holds the members.
    state $copy_arg_to_index = {
        trace_file_handle => Marpa::R2::Internal::Scanless::G::TRACE_FILE_HANDLE
    };

    ARG: for my $arg_name ( keys %flat_args ) {
        my $index = $copy_arg_to_index->{$arg_name};
        next ARG if not defined $index;
        my $value = $flat_args{$arg_name};
        $slg->[$index] = $value;
    } ## end ARG: for my $arg_name ( keys %flat_args )

    # Trace file handle needs to be populated downwards
    if ( defined( my $trace_file_handle = $flat_args{trace_file_handle} ) ) {
        GRAMMAR:
        for my $naif_grammar (
            $slg->[Marpa::R2::Internal::Scanless::G::THICK_G1_GRAMMAR],
            @{ $slg->[Marpa::R2::Internal::Scanless::G::THICK_LEX_GRAMMARS] }
            )
        {
            next GRAMMAR if not defined $naif_grammar;
            $naif_grammar->set( { trace_file_handle => $trace_file_handle } );
        } ## end GRAMMAR: for my $naif_grammar ( $slg->[...])
    } ## end if ( defined( my $trace_file_handle = $flat_args{...}))

    if ( $method eq 'new' or $method eq 'meta' ) {

        # Prune flat args of all those named args which are NOT to be copied
        # into the NAIF recce args
        for my $arg_name ( keys %flat_args ) {
            delete $flat_args{$arg_name}
                if not $copy_to_g1_args->{$arg_name};
        }

        # trace file handle must always be defined
        $slg->[Marpa::R2::Internal::Scanless::G::TRACE_FILE_HANDLE] //= \*STDERR;

        return ($dsl, \%flat_args);
    } ## end if ( $method eq 'new' )

    return;
<commit_msg>Work on SLIF grammar args: t+<commit_after># Copyright 2013 <NAME>
# This file is part of Marpa::R2.  Marpa::R2 is free software: you can
# redistribute it and/or modify it under the terms of the GNU Lesser
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later version.
#
# Marpa::R2 is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser
# General Public License along with Marpa::R2.  If not, see
# http://www.gnu.org/licenses/.

package Marpa::R2::Scanless::G;

use 5.010;
use strict;
use warnings;

use vars qw($VERSION $STRING_VERSION);
$VERSION        = '2.077_014';
$STRING_VERSION = $VERSION;
## no critic(BuiltinFunctions::ProhibitStringyEval)
$VERSION = eval $VERSION;
## use critic

package Marpa::R2::Internal::Scanless::G;

use Scalar::Util 'blessed';
use English qw( -no_match_vars );

# names of packages for strings
our $PACKAGE = 'Marpa::R2::Scanless::G';

sub Marpa::R2::Internal::Scanless::meta_grammar {

    my $meta_slg = bless [], 'Marpa::R2::Scanless::G';
    state $hashed_metag = Marpa::R2::Internal::MetaG::hashed_grammar();
    Marpa::R2::Internal::Scanless::G::hash_to_runtime( $meta_slg,
        $hashed_metag,
        { bless_package => 'Marpa::R2::Internal::MetaAST_Nodes' } );

    my $thick_g1_grammar =
        $meta_slg->[Marpa::R2::Internal::Scanless::G::THICK_G1_GRAMMAR];
    my @mask_by_rule_id;
    $mask_by_rule_id[$_] = $thick_g1_grammar->_rule_mask($_)
        for $thick_g1_grammar->rule_ids();
    $meta_slg->[Marpa::R2::Internal::Scanless::G::MASK_BY_RULE_ID] =
        \@mask_by_rule_id;

    return $meta_slg;

} ## end sub Marpa::R2::Internal::Scanless::meta_grammar

sub Marpa::R2::Scanless::G::new {
    my ( $class, @hash_ref_args ) = @_;

    my $slg = [];
    bless $slg, $class;

    my ($dsl, $g1_args) = Marpa::R2::Internal::Scanless::G::set ( $slg, 'new', @hash_ref_args );
    my $ast = Marpa::R2::Internal::MetaAST->new( $dsl );
    my $hashed_ast = $ast->ast_to_hash();
    Marpa::R2::Internal::Scanless::G::hash_to_runtime($slg, $hashed_ast, $g1_args);
    return $slg;
} ## end sub Marpa::R2::Scanless::G::new

sub Marpa::R2::Scanless::G::set {
    my ( $slg, @hash_ref_args ) = @_;
    Marpa::R2::Internal::Scanless::G::set ( $slg, 'set', @hash_ref_args );
    return $slg;
}

# The context flag indicates whether this ::set() is called directly by the user;
# is for the external constructor; or is for the internal ("meta") constructor.
# "Context" flags of this kind
# are much decried practice, and for good reason, but in this case
# I think it is justified.
# This logic really needs to be all in one place, and so a flag
# to trigger the minor differences needed by the various calling
# contexts is a small price to pay.
sub Marpa::R2::Internal::Scanless::G::set {
    my ( $slg, $method, @hash_ref_args ) = @_;

    # Other possible grammar options:
    # default_rank
    # inaccessible_ok
    # unproductive_ok
    # warnings

    state $copy_to_g1_args =
        { map { ( $_, 1 ); }
            qw(trace_file_handle action_object default_action bless_package) };
    state $set_method_args =
        { map { ( $_, 1 ); } qw(trace_file_handle) };
    state $new_method_args = {
        map { ( $_, 1 ); } qw(source), keys %{$copy_to_g1_args}
    };
    for my $args (@hash_ref_args) {
        my $ref_type = ref $args;
        if ( not $ref_type ) {
            Marpa::R2::exception( q{$slr->}
                    . $method
                    . qq{() expects args as ref to HASH; got non-reference instead}
            );
        } ## end if ( not $ref_type )
        if ( $ref_type ne 'HASH' ) {
            Marpa::R2::exception( q{$slr->}
                    . $method
                    . qq{() expects args as ref to HASH, got ref to $ref_type instead}
            );
        } ## end if ( $ref_type ne 'HASH' )
    } ## end for my $args (@hash_ref_args)

    my %flat_args = ();
    for my $hash_ref (@hash_ref_args) {
        ARG: for my $arg_name ( keys %{$hash_ref} ) {
            $flat_args{$arg_name} = $hash_ref->{$arg_name};
        }
    }

    my $ok_args = $set_method_args;
    $ok_args = $new_method_args if $method eq 'new';
    my @bad_args = grep { not $ok_args->{$_} } keys %flat_args;
    if ( scalar @bad_args ) {
        Marpa::R2::exception(
            q{Bad named argument(s) to $slr->}
                . $method
                . q{() method: }
                . join q{ },
            @bad_args
        );
    } ## end if ( scalar @bad_args )

    my $dsl;
    if ( $method eq 'new' ) {
        state $arg_name = 'source';
        $dsl = $flat_args{$arg_name};
        Marpa::R2::exception(
            qq{Marpa::R2::Scanless::G::new() called without a "$arg_name" argument}
        ) if not defined $dsl;
        my $ref_type = ref $dsl;
        if ( $ref_type ne 'SCALAR' ) {
            my $desc = $ref_type ? "a ref to $ref_type" : 'not a ref';
            Marpa::R2::exception(
                qq{'$arg_name' name argument to Marpa::R2::Scanless::G->new() is $desc\n},
                "  It should be a ref to a string\n"
            );
        } ## end if ( $ref_type ne 'SCALAR' )
    } ## end if ( $method eq 'new' )

    # A bit hack-ish, but some named args will be copies straight to a member of
    # the Scanless::G class, so this maps named args to the index of the array
    # that holds the members.
    state $copy_arg_to_index = {
        trace_file_handle => Marpa::R2::Internal::Scanless::G::TRACE_FILE_HANDLE
    };

    ARG: for my $arg_name ( keys %flat_args ) {
        my $index = $copy_arg_to_index->{$arg_name};
        next ARG if not defined $index;
        my $value = $flat_args{$arg_name};
        $slg->[$index] = $value;
    } ## end ARG: for my $arg_name ( keys %flat_args )

    # Trace file handle needs to be populated downwards
    if ( defined( my $trace_file_handle = $flat_args{trace_file_handle} ) ) {
        GRAMMAR:
        for my $naif_grammar (
            $slg->[Marpa::R2::Internal::Scanless::G::THICK_G1_GRAMMAR],
            @{ $slg->[Marpa::R2::Internal::Scanless::G::THICK_LEX_GRAMMARS] }
            )
        {
            next GRAMMAR if not defined $naif_grammar;
            $naif_grammar->set( { trace_file_handle => $trace_file_handle } );
        } ## end GRAMMAR: for my $naif_grammar ( $slg->[...])
    } ## end if ( defined( my $trace_file_handle = $flat_args{...}))

    if ( $method eq 'new' ) {

        # Prune flat args of all those named args which are NOT to be copied
        # into the NAIF recce args
        for my $arg_name ( keys %flat_args ) {
            delete $flat_args{$arg_name}
                if not $copy_to_g1_args->{$arg_name};
        }

        # trace file handle must always be defined
        $slg->[Marpa::R2::Internal::Scanless::G::TRACE_FILE_HANDLE] //= \*STDERR;

        return ($dsl, \%flat_args);
    } ## end if ( $method eq 'new' )

    return;

{'fraction_non_alphanumeric': 0.13927710843373495, 'fraction_numerical': 0.00874354561101549, 'mean_word_length': 2.7037225905150435, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7186602', 'n_tokens_mistral': 5065, 'n_tokens_neox': 4915, 'n_words': 1343}