<commit_before>#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <genericLogger.h>
#include <marpaESLIF.h>

static char context;

static short                         inputReaderb(void *userDatavp, char **inputsp, size_t *inputlp, short *eofbp, short *characterStreambp, char **encodingsp, size_t *encodinglp);
static short                         doparseb(genericLogger_t *genericLoggerp, marpaESLIFRecognizer_t *marpaESLIFRecognizerParentp, marpaESLIFGrammar_t *marpaESLIFGrammarObjectp, char *inputs, int recursionleveli);

typedef struct marpaESLIFTester_context {
  char            *inputs;
  size_t           inputl;
} marpaESLIFTester_context_t;

const static char *dsl = "\n"
":default ::= action => ::shift\n"
":start       ::= XXXXXX # Replaced on-the-fly by json or object\n"
"json         ::= object\n"
"               | array\n"
"# comma is provided as a char class here, to ensure that char classes\n"
"# as separators are in the test suite.\n"
"object       ::= LCURLY members RCURLY\n"
"               | OBJECT_FROM_INNER_GRAMMAR\n"
"members      ::= pair*                 action => do_array separator => ','\n"
"pair         ::= string ':' value      action => do_array\n"
"value        ::= string\n"
"               | object\n"
"               | number\n"
"               | array\n"
"               | 'true'                action => do_true\n"
"               | 'false'               action => do_true\n"
"               | 'null'                action => ::undef\n"
"array        ::= '[' ']'               action => do_empty_array\n"
"               | '[' elements ']'\n"
"# comma is provided as a char class here, to ensure that char classes\n"
"# as separators are in the test suite.\n"
"elements     ::= value+                action => do_array separator => ','\n"
"number         ~ int\n"
"               | int frac\n"
"               | int exp\n"
"               | int frac exp\n"
"int            ~ digits\n"
"               | '-' digits\n"
"digits         ~ [\\d]+\n"
"frac           ~ '.' digits\n"
"exp            ~ e digits\n"
"e              ~ 'e'\n"
"               | 'e+'\n"
"               | 'e-'\n"
"               | 'E'\n"
"               | 'E+'\n"
"               | 'E-'\n"
"string       ::= lstring\n"
":lexeme ::= lstring pause => after event => lstring$\n"
"lstring        ~ quote in_string quote\n"
"quote          ~ [\"]\n"
"in_string      ~ in_string_char*\n"
"in_string_char  ~ [^\"] | '\\\\\"'\n"
":discard       ::= whitespace\n"
"whitespace     ~ [\\s]+\n"
":lexeme ::= LCURLY pause => before event => ^LCURLY\n"
"LCURLY         ~ '{'\n"
":lexeme ::= RCURLY pause => before event => ^RCURLY\n"
"RCURLY         ~ '}'\n"
"OBJECT_FROM_INNER_GRAMMAR ~ [^\\s\\S]\n"
  ;

static char *inputs[] = {
    "\xEF\xBB\xBF{\"test\":\"1\"}",
    "{\"test\":[1,2,3]}",
    "{\"test\":true}",
    "{\"test\":false}",
    "{\"test\":null}",
    "{\"test\":null, \"test2\":\"hello world\"}",
    "{\"test\":\"1.25\"}",
    "{\"test\":\"1.25e4\"}",
    "[]",
    "[\n"
    "  {\n"
    "     \"precision\": \"zip\",\n"
    "     \"Latitude\":  37.7668,\n"
    "     \"Longitude\": -122.3959,\n"
    "     \"Address\":   \"\",\n"
    "     \"City\":      \"SAN FRANCISCO\",\n"
    "     \"State\":     \"CA\",\n"
    "     \"Zip\":       \"94107\",\n"
    "     \"Country\":   \"US\"\n"
    "  },\n"<commit_msg>test/jsonTesterWithSharedStream.c: add empty string test [ci skip]<commit_after>#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <genericLogger.h>
#include <marpaESLIF.h>

static char context;

static short                         inputReaderb(void *userDatavp, char **inputsp, size_t *inputlp, short *eofbp, short *characterStreambp, char **encodingsp, size_t *encodinglp);
static short                         doparseb(genericLogger_t *genericLoggerp, marpaESLIFRecognizer_t *marpaESLIFRecognizerParentp, marpaESLIFGrammar_t *marpaESLIFGrammarObjectp, char *inputs, int recursionleveli);

typedef struct marpaESLIFTester_context {
  char            *inputs;
  size_t           inputl;
} marpaESLIFTester_context_t;

const static char *dsl = "\n"
":default ::= action => ::shift\n"
":start       ::= XXXXXX # Replaced on-the-fly by json or object\n"
"json         ::= object\n"
"               | array\n"
"# comma is provided as a char class here, to ensure that char classes\n"
"# as separators are in the test suite.\n"
"object       ::= LCURLY members RCURLY\n"
"               | OBJECT_FROM_INNER_GRAMMAR\n"
"members      ::= pair*                 action => do_array separator => ','\n"
"pair         ::= string ':' value      action => do_array\n"
"value        ::= string\n"
"               | object\n"
"               | number\n"
"               | array\n"
"               | 'true'                action => do_true\n"
"               | 'false'               action => do_true\n"
"               | 'null'                action => ::undef\n"
"array        ::= '[' ']'               action => do_empty_array\n"
"               | '[' elements ']'\n"
"# comma is provided as a char class here, to ensure that char classes\n"
"# as separators are in the test suite.\n"
"elements     ::= value+                action => do_array separator => ','\n"
"number         ~ int\n"
"               | int frac\n"
"               | int exp\n"
"               | int frac exp\n"
"int            ~ digits\n"
"               | '-' digits\n"
"digits         ~ [\\d]+\n"
"frac           ~ '.' digits\n"
"exp            ~ e digits\n"
"e              ~ 'e'\n"
"               | 'e+'\n"
"               | 'e-'\n"
"               | 'E'\n"
"               | 'E+'\n"
"               | 'E-'\n"
"string       ::= lstring\n"
":lexeme ::= lstring pause => after event => lstring$\n"
"lstring        ~ quote in_string quote\n"
"quote          ~ [\"]\n"
"in_string      ~ in_string_char*\n"
"in_string_char  ~ [^\"] | '\\\\\"'\n"
":discard       ::= whitespace\n"
"whitespace     ~ [\\s]+\n"
":lexeme ::= LCURLY pause => before event => ^LCURLY\n"
"LCURLY         ~ '{'\n"
":lexeme ::= RCURLY pause => before event => ^RCURLY\n"
"RCURLY         ~ '}'\n"
"OBJECT_FROM_INNER_GRAMMAR ~ [^\\s\\S]\n"
  ;

static char *inputs[] = {
    "\xEF\xBB\xBF{\"test\":\"1\"}",
    "{\"test\":[1,2,3]}",
    "{\"test\":true}",
    "{\"test\":false}",
    "{\"test\":null}",
    "{\"test\":null, \"test2\":\"hello world\"}",
    "{\"test\":\"1.25\"}",
    "{\"test\":\"1.25e4\"}",
    "{\"empty string\":\"\"}",
    "[]",
    "[\n"
    "  {\n"
    "     \"precision\": \"zip\",\n"
    "     \"Latitude\":  37.7668,\n"
    "     \"Longitude\": -122.3959,\n"
    "     \"Address\":   \"\",\n"
    "     \"City\":      \"SAN FRANCISCO\",\n"
    "     \"State\":     \"CA\",\n"
    "     \"Zip\":       \"94107\",\n"
    "     \"Country\":   \"US\"\n"
    "  },\n"
{'fraction_non_alphanumeric': 0.20026369762672136, 'fraction_numerical': 0.008789920890711983, 'mean_word_length': 1.7362725450901804, 'pattern_counts': {'":': 47, '<': 15, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '107322', 'n_tokens_mistral': 2491, 'n_tokens_neox': 2413, 'n_words': 507}