<commit_before>package MyRecognizerInterface;
use strict;
use diagnostics;

sub new                    { my ($pkg, $characterStream) = @_; bless { characterStream => $characterStream }, $pkg }
sub read                   { 1 }
sub isEof                  { 1 }
sub isCharacterStream      { $_[0]->{characterStream} // 1 }
sub encoding               { }
sub data                   { " " } # One byte
sub isWithDisableThreshold { 0 }
sub isWithExhaustion       { 0 }
sub isWithNewline          { 1 }
sub isWithTrack            { 1 }

package MyValueInterface;
use strict;
use diagnostics;

sub new                { my ($pkg) = @_; bless { result => undef }, $pkg }
sub isWithHighRankOnly { 1 }
sub isWithOrderByRank  { 1 }
sub isWithAmbiguous    { 0 }
sub isWithNull         { 0 }
sub maxParses          { 0 }
sub getResult          { $_[0]->{result} }
sub setResult          { $_[0]->{result} = $_[1] }
sub perl_proxy         { $_[1] }

package main;
use strict;
use warnings FATAL => 'all';
use Test::Deep qw/cmp_details deep_diag/;
use Log::Log4perl qw/:easy/;
use Log::Any::Adapter;
use Log::Any qw/$log/;
use Math::BigInt;
use Math::BigFloat;

our @input;
BEGIN {
    use utf8;
    @input =
        (
         undef,
         "XXX",
         "\xa0\xa1",
         ["\xa0\xa1", 0],
         ["\xf0\x28\x8c\x28", 0],
         "Ḽơᶉëᶆ ȋṕšᶙṁ ḍỡḽǭᵳ ʂǐť ӓṁệẗ, ĉṓɲṩḙċťᶒțûɾ ấɖḯƥĭṩčįɳġ ḝłįʈ, șếᶑ ᶁⱺ ẽḭŭŝḿꝋď ṫĕᶆᶈṓɍ ỉñḉīḑȋᵭṵńť ṷŧ ḹẩḇőꝛế éȶ đꝍꞎôꝛȇ ᵯáꞡᶇā ąⱡîɋṹẵ.",
         ["Ḽơᶉëᶆ ȋṕšᶙṁ ḍỡḽǭᵳ ʂǐť ӓṁệẗ, ĉṓɲṩḙċťᶒțûɾ ấɖḯƥĭṩčįɳġ ḝłįʈ, șếᶑ ᶁⱺ ẽḭŭŝḿꝋď ṫĕᶆᶈṓɍ ỉñḉīḑȋᵭṵńť ṷŧ ḹẩḇőꝛế éȶ đꝍꞎôꝛȇ ᵯáꞡᶇā ąⱡîɋṹẵ.", 0],
         ["Ḽơᶉëᶆ ȋṕšᶙṁ ḍỡḽǭᵳ ʂǐť ӓṁệẗ, ĉṓɲṩḙċťᶒțûɾ ấɖḯƥĭṩčįɳġ ḝłįʈ, șếᶑ ᶁⱺ ẽḭŭŝḿꝋď ṫĕᶆᶈṓɍ ỉñḉīḑȋᵭṵńť ṷŧ ḹẩḇőꝛế éȶ đꝍꞎôꝛȇ ᵯáꞡᶇā ąⱡîɋṹẵ.", 1],
         0,
         1,
         -32768,
         32767,
         -32769,
         32768,
         2.34,
         1.6e+308,
         Math::BigFloat->new("6.78E+9"),
         Math::BigInt->new("6.78E+9"),
         [[1, undef, 2], 0],
         ""
        );
}
use Safe::Isa;
use Test::More tests => 1 + scalar(@input) + 4; # 4 pushes after the require_ok
use Test::More::UTF8;
use open qw( :utf8 :std );

BEGIN { require_ok('MarpaX::ESLIF') }
push(@input, $MarpaX::ESLIF::true);
push(@input, $MarpaX::ESLIF::false);
push(@input, { one => "one", two => "two", perltrue => 1, true => $MarpaX::ESLIF::true, false => $MarpaX::ESLIF::false, 'else' => 'again', 'undef' => undef }); # will cause trouble because natively lua discards it
push(@input, MarpaX::ESLIF::String->new("XXXḼơᶉëᶆYYY", 'UTF-8'));
#
# Init log
#
our $defaultLog4perlConf = '
log4perl.rootLogger              = TRACE, Screen
log4perl.appender.Screen         = Log::Log4perl::Appender::Screen
log4perl.appender.Screen.stderr  = 0
log4perl.appender.Screen.layout  = PatternLayout
log4perl.appender.Screen.layout.ConversionPattern = %d %-5p %6P %m{chomp}%n
';
Log::Log4perl::init(\$defaultLog4perlConf);
Log::Any::Adapter->set('Log4perl');

my $grammar = q{
event ^perl_input = predicted perl_input

perl_output ::= lua_proxy  action => perl_proxy
lua_proxy   ::= perl_input action => ::lua->lua_proxy
perl_input  ::= PERL_INPUT action => perl_proxy<commit_msg>t/import_export.t: Removed obsolete comment [ci skip]<commit_after>package MyRecognizerInterface;
use strict;
use diagnostics;

sub new                    { my ($pkg, $characterStream) = @_; bless { characterStream => $characterStream }, $pkg }
sub read                   { 1 }
sub isEof                  { 1 }
sub isCharacterStream      { $_[0]->{characterStream} // 1 }
sub encoding               { }
sub data                   { " " } # One byte
sub isWithDisableThreshold { 0 }
sub isWithExhaustion       { 0 }
sub isWithNewline          { 1 }
sub isWithTrack            { 1 }

package MyValueInterface;
use strict;
use diagnostics;

sub new                { my ($pkg) = @_; bless { result => undef }, $pkg }
sub isWithHighRankOnly { 1 }
sub isWithOrderByRank  { 1 }
sub isWithAmbiguous    { 0 }
sub isWithNull         { 0 }
sub maxParses          { 0 }
sub getResult          { $_[0]->{result} }
sub setResult          { $_[0]->{result} = $_[1] }
sub perl_proxy         { $_[1] }

package main;
use strict;
use warnings FATAL => 'all';
use Test::Deep qw/cmp_details deep_diag/;
use Log::Log4perl qw/:easy/;
use Log::Any::Adapter;
use Log::Any qw/$log/;
use Math::BigInt;
use Math::BigFloat;

our @input;
BEGIN {
    use utf8;
    @input =
        (
         undef,
         "XXX",
         "\xa0\xa1",
         ["\xa0\xa1", 0],
         ["\xf0\x28\x8c\x28", 0],
         "Ḽơᶉëᶆ ȋṕšᶙṁ ḍỡḽǭᵳ ʂǐť ӓṁệẗ, ĉṓɲṩḙċťᶒțûɾ ấɖḯƥĭṩčįɳġ ḝłįʈ, șếᶑ ᶁⱺ ẽḭŭŝḿꝋď ṫĕᶆᶈṓɍ ỉñḉīḑȋᵭṵńť ṷŧ ḹẩḇőꝛế éȶ đꝍꞎôꝛȇ ᵯáꞡᶇā ąⱡîɋṹẵ.",
         ["Ḽơᶉëᶆ ȋṕšᶙṁ ḍỡḽǭᵳ ʂǐť ӓṁệẗ, ĉṓɲṩḙċťᶒțûɾ ấɖḯƥĭṩčįɳġ ḝłįʈ, șếᶑ ᶁⱺ ẽḭŭŝḿꝋď ṫĕᶆᶈṓɍ ỉñḉīḑȋᵭṵńť ṷŧ ḹẩḇőꝛế éȶ đꝍꞎôꝛȇ ᵯáꞡᶇā ąⱡîɋṹẵ.", 0],
         ["Ḽơᶉëᶆ ȋṕšᶙṁ ḍỡḽǭᵳ ʂǐť ӓṁệẗ, ĉṓɲṩḙċťᶒțûɾ ấɖḯƥĭṩčįɳġ ḝłįʈ, șếᶑ ᶁⱺ ẽḭŭŝḿꝋď ṫĕᶆᶈṓɍ ỉñḉīḑȋᵭṵńť ṷŧ ḹẩḇőꝛế éȶ đꝍꞎôꝛȇ ᵯáꞡᶇā ąⱡîɋṹẵ.", 1],
         0,
         1,
         -32768,
         32767,
         -32769,
         32768,
         2.34,
         1.6e+308,
         Math::BigFloat->new("6.78E+9"),
         Math::BigInt->new("6.78E+9"),
         [[1, undef, 2], 0],
         ""
        );
}
use Safe::Isa;
use Test::More tests => 1 + scalar(@input) + 4; # 4 pushes after the require_ok
use Test::More::UTF8;
use open qw( :utf8 :std );

BEGIN { require_ok('MarpaX::ESLIF') }
push(@input, $MarpaX::ESLIF::true);
push(@input, $MarpaX::ESLIF::false);
push(@input, { one => "one", two => "two", perltrue => 1, true => $MarpaX::ESLIF::true, false => $MarpaX::ESLIF::false, 'else' => 'again', 'undef' => undef });
push(@input, MarpaX::ESLIF::String->new("XXXḼơᶉëᶆYYY", 'UTF-8'));
#
# Init log
#
our $defaultLog4perlConf = '
log4perl.rootLogger              = TRACE, Screen
log4perl.appender.Screen         = Log::Log4perl::Appender::Screen
log4perl.appender.Screen.stderr  = 0
log4perl.appender.Screen.layout  = PatternLayout
log4perl.appender.Screen.layout.ConversionPattern = %d %-5p %6P %m{chomp}%n
';
Log::Log4perl::init(\$defaultLog4perlConf);
Log::Any::Adapter->set('Log4perl');

my $grammar = q{
event ^perl_input = predicted perl_input

perl_output ::= lua_proxy  action => perl_proxy
lua_proxy   ::= perl_input action => ::lua->lua_proxy
perl_input  ::= PERL_INPUT action => perl_proxy
{'fraction_non_alphanumeric': 0.14431023521932612, 'fraction_numerical': 0.029879211697393517, 'mean_word_length': 2.7258732978093545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4993652', 'n_tokens_mistral': 3426, 'n_tokens_neox': 3223, 'n_words': 562}