<commit_before>package MarpaX::JSON;

use 5.010;
use strict;
use warnings;

use Marpa::R2 2.047_011;

sub new {
    my ($class) = @_;

    my $self = bless {}, $class;

    $self->{grammar} = Marpa::R2::Scanless::G->new(
        {
            action_object  => 'MarpaX::JSON::Actions',
	    bless_package => 'My_Nodes',
            source         => \(<<'END_OF_SOURCE'),
:default ::= action => ::array

:start       ::= json

json         ::= object action => ::first
               | array action => ::first

object       ::= ('{') members ('}') bless => object

members      ::= pair*                 separator => <comma>

pair         ::= string (':') value

value        ::= string action => ::first
               | object action => ::first
               | number action => ::first
               | array action => ::first
               | 'true' action => ::first
               | 'false' action => ::first
               | 'null' action => ::first


array        ::= ('[' ']')
               | ('[') elements (']') action => ::first

elements     ::= value+                separator => <comma>

number         ~ int
               | int frac
               | int exp
               | int frac exp

int            ~ digits
               | '-' digits

digits         ~ [\d]+

frac           ~ '.' digits

exp            ~ e digits

e              ~ 'e'
               | 'e+'
               | 'e-'
               | 'E'
               | 'E+'
               | 'E-'

string ::= lstring bless => string
 
lstring ~ quote in_string quote
quote ~ ["]
  
in_string ~ in_string_char*
   
in_string_char ~ [^"\\]
   | '\' '"'
   | '\' 'b'
   | '\' 'f'
   | '\' 't'
   | '\' 'n'
   | '\' 'r'
   | '\' 'u' four_hex_digits
   | '\' '/'
   | '\\'
    
four_hex_digits ~ hex_digit hex_digit hex_digit hex_digit
hex_digit ~ [0-9a-fA-F]
     
comma          ~ ','

:discard       ~ whitespace
whitespace     ~ [\s]+

END_OF_SOURCE

        }
    );
    return $self;
}

sub parse {
    my ($self, $string) = @_;
    my $re = Marpa::R2::Scanless::R->new( { grammar => $self->{grammar} } );
    my $length = length $string;
    my $pos = $re->read( \$string );
    die "Read short of end: $pos vs. $length" if $pos < $length;
    my $value_ref = $re->value();
    return ${$value_ref};
}

sub parse_json {
    my ($string) = @_;
    my $parser = MarpaX::JSON->new();<commit_msg>Work on JSON parser<commit_after>package MarpaX::JSON;

use 5.010;
use strict;
use warnings;

use Marpa::R2 2.047_011;

sub new {
    my ($class) = @_;

    my $self = bless {}, $class;

    $self->{grammar} = Marpa::R2::Scanless::G->new(
        {
            action_object  => 'MarpaX::JSON::Actions',
	    bless_package => 'My_Nodes',
            source         => \(<<'END_OF_SOURCE'),
:default ::= action => ::array

:start       ::= json

json         ::= object action => ::first
               | array action => ::first

object       ::= ('{') members ('}') bless => hash

members      ::= pair*                 separator => <comma>

pair         ::= string (':') value

value        ::= string action => ::first
               | object action => ::first
               | number action => ::first
               | array action => ::first
               | 'true' action => ::first
               | 'false' action => ::first
               | 'null' action => ::first


array        ::= ('[' ']')
               | ('[') elements (']') action => ::first

elements     ::= value+                separator => <comma>

number         ~ int
               | int frac
               | int exp
               | int frac exp

int            ~ digits
               | '-' digits

digits         ~ [\d]+

frac           ~ '.' digits

exp            ~ e digits

e              ~ 'e'
               | 'e+'
               | 'e-'
               | 'E'
               | 'E+'
               | 'E-'

string ::= lstring bless => string
 
lstring ~ quote in_string quote
quote ~ ["]
  
in_string ~ in_string_char*
   
in_string_char ~ [^"\\]
   | '\' '"'
   | '\' 'b'
   | '\' 'f'
   | '\' 't'
   | '\' 'n'
   | '\' 'r'
   | '\' 'u' four_hex_digits
   | '\' '/'
   | '\\'
    
four_hex_digits ~ hex_digit hex_digit hex_digit hex_digit
hex_digit ~ [0-9a-fA-F]
     
comma          ~ ','

:discard       ~ whitespace
whitespace     ~ [\s]+

END_OF_SOURCE

        }
    );
    return $self;
}

sub eval_json {
    my ($thing) = @_;
    my $type = ref $thing;
    say STDERR $type;
    if ($type eq 'REF') {
        return \eval_json(${$thing});
    }
    if ($type eq 'ARRAY') {
        return [ map { eval_json($_) } @{$thing} ];
    }
    if ($type eq 'My_Nodes::string') {
        my $string = substr $thing->[0], 1, -1;
	return decode_string($string) if (index $string, '\\') >= 0;
	return $string;
    }
    if ($type eq 'My_Nodes::hash') {
	return { map { eval_json($_->[0]), eval_json($_->[1]) } @{$thing->[0]} };
    }
    return $thing;
}

sub parse {
    my ($self, $string) = @_;
    my $re = Marpa::R2::Scanless::R->new( { grammar => $self->{grammar} } );
    my $length = length $string;
    my $pos = $re->read( \$string );
    die "Read short of end: $pos vs. $length" if $pos < $length;
    my $value_ref = $re->value();
    die "Parse failed" if not defined $value_ref;
    my $value = eval_json($value_ref);
    return $value;
}

sub parse_json {
    my ($string) = @_;
    my $parser = MarpaX::JSON->new();
{'fraction_non_alphanumeric': 0.18195348837209302, 'fraction_numerical': 0.007255813953488372, 'mean_word_length': 1.6186069167072576, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '737630', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1794, 'n_words': 428}