<commit_before>#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <genericLogger.h>
#include <marpaESLIF.h>

static short inputReaderb(void *userDatavp, char **inputsp, size_t *inputlp, short *eofbp, short *characterStreambp, char **encodingsp, size_t *encodinglp);

typedef struct marpaESLIFTester_context {
  genericLogger_t *genericLoggerp;
  char            *inputs;
  size_t           inputl;
} marpaESLIFTester_context_t;

const static char *dsl = "\n"
  "#\n"
  "# Default action is to propagate the first RHS value\n"
  "#\n"
  ":default ::= action => ::shift\n"
  "\n"
  "                   #######################################################\n"
  "                   # >>>>>>>>>>>>>>>> Strict JSON Grammar <<<<<<<<<<<<<<<<\n"
  "                   #######################################################\n"
  "\n"
  "# ----------------\n"
  "# Start is a value\n"
  "# ----------------\n"
  ":start ::= value\n"
  "\n"
  "# -------------------\n"
  "# Composite separator\n"
  "# -------------------\n"
  "comma    ::= ','                                  action         => ::undef   # No-op anyway, override ::shift (default action)\n"
  "\n"
  "# ----------\n"
  "# JSON value\n"
  "# ----------\n"
  "value    ::= string                                                           # ::shift (default action)\n"
  "           | number                                                           # ::shift (default action)\n"
  "           | object                                                           # ::shift (default action)\n"
  "           | array                                                            # ::shift (default action)\n"
  "           | 'true'                               action         => ::lua->lua_true      # Returns a lua true value\n"
  "           | 'false'                              action         => ::lua->lua_false     # Returns a lua false value\n"
  "           | 'null'\n"
  "\n"
  "# -----------\n"
  "# JSON object\n"
  "# -----------\n"
  "object   ::= '{' inc members '}' dec              action         => ::copy[2] # Returns members\n"
  "members  ::= pairs*                               action         => ::lua->lua_members   # Returns { @{pairs1}, ..., @{pair2} }\n"
  "                                                  separator      => comma     # ... separated by comma\n"
  "                                                  proper         => 1         # ... with no trailing separator\n"
  "                                                  hide-separator => 1         # ... and hide separator in the action\n"
  "                                                  \n"
  "pairs    ::= string ':' value                     action         => ::lua->lua_pairs     # Returns [ string, value ]\n"
  "\n"
  "# -----------\n"
  "# JSON Arrays\n"
  "# -----------\n"
  "array    ::= '[' inc elements ']' dec             action         => ::copy[2] # Returns elements\n"
  "elements ::= value*                               action => ::lua->lua_elements          # Returns [ value1, ..., valuen ]\n"
  "                                                  separator      => comma     # ... separated by comma\n"
  "                                                  proper         => 1         # ... with no trailing separator\n"
  "                                                  hide-separator => 1         # ... and hide separator in the action\n"
  "                                                  \n"
  "\n"
  "# ------------\n"
  "# JSON Numbers\n"
  "# ------------\n"
  "number ::= NUMBER            # /* bignum */       action => ::lua->lua_number            # Prepare for eventual bignum extension\n"
  "\n"
  "NUMBER   ~ _INT\n"
  "         | _INT _FRAC\n"
  "         | _INT _EXP\n"
  "         | _INT _FRAC _EXP\n"
  "_INT     ~ _DIGIT\n"
  "         | _DIGIT19 _DIGITS\n"
  "         | '-' _DIGIT\n"
  "         | '-' _DIGIT19 _DIGITS\n"
  "_DIGIT   ~ [0-9]\n"
  "_DIGIT19 ~ [1-9]\n"
  "_FRAC    ~ '.' _DIGITS\n"
  "_EXP     ~ _E _DIGITS\n"
  "_DIGITS  ~ _DIGIT+\n"
  "_E       ~ /e[+-]?/i\n"
  "\n"
  "# -----------\n"
  "# JSON String\n"
  "# -----------\n"
  "string     ::= '\"' discardOff chars '\"' discardOn action => ::copy[2]               # Only chars is of interest\n"
  "discardOff ::=                                    action => ::undef                 # Nullable rule used to disable discard\n"
  "discardOn  ::=                                    action => ::undef                 # Nullable rule used to enable discard\n"
  "\n"
  "event :discard[on]  = nulled discardOn                                                           # Implementation of discard disabing using reserved ':discard[on]' keyword\n"
  "event :discard[off] = nulled discardOff                                                          # Implementation of discard enabling using reserved ':discard[off]' keyword\n"
  "\n"
  "chars   ::= filled                                                                               # ::shift (default action)\n"
  "filled  ::= char+                                 action => ::concat                # Returns join('', char1, ..., charn)\n"
  "chars   ::=                                       action => ::lua->lua_empty_string            # Prefering empty string instead of undef\n"
  "char    ::= [^\"\\\\\\x00-\\x1F]                                                         # ::shift (default action) - take care PCRE2 [:cntrl:] includes DEL character\n"
  "          | '\\\\' '\"'                              action => ::copy[1]               # Returns double quote, already ok in data\n"
  "          | '\\\\' '\\\\'                             action => ::copy[1]               # Returns backslash, already ok in data\n"
  "          | '\\\\' '/'                              action => ::copy[1]               # Returns slash, already ok in data\n"
  "          | '\\\\' 'b'                              action => ::u8\"\\x{08}\"\n"
  "          | '\\\\' 'f'                              action => ::u8\"\\x{0C}\"\n"
  "          | '\\\\' 'n'                              action => ::u8\"\\x{0A}\"\n"
  "          | '\\\\' 'r'                              action => ::u8\"\\x{0D}\"\n"
  "          | '\\\\' 't'                              action => ::u8\"\\x{09}\"\n"
  "          | /(?:\\\\u[[:xdigit:]]{4})+/             action => ::lua->lua_unicode\n"
  "\n"
  "\n"
  "# -------------------------\n"
  "# Unsignificant whitespaces\n"
  "# -------------------------\n"
  ":discard ::= /[\\x{9}\\x{A}\\x{D}\\x{20}]+/\n"
  "\n"
  "# ------------------------------------------------------\n"
  "# Needed for eventual depth extension - no op by default\n"
  "# ------------------------------------------------------\n"
  "inc ::=                                                        action => ::undef\n"
  "dec ::=                                                        action => ::undef\n"
  "\n"
  "                   #######################################################\n"
  "                   # >>>>>>>>>>>>>>>>>> JSON Extensions <<<<<<<<<<<<<<<<<<\n"
  "                   #######################################################\n"
  "\n"
  "# --------------------------\n"
  "# Unlimited commas extension\n"
  "# --------------------------\n"
  "# /* Unlimited commas */commas   ::= comma+\n"
  "\n"
  "# --------------------------\n"
  "# Perl comment extension\n"
  "# --------------------------\n"
  "# /* Perl comment */:discard ::= /(?:(?:#)(?:[^\\n]*)(?:\\n|\\z))/u\n"
  "\n"
  "# --------------------------\n"
  "# C++ comment extension\n"
  "# --------------------------\n"
  "# /* C++ comment */:discard ::= /(?:(?:(?:\\/\\/)(?:[^\\n]*)(?:\\n|\\z))|(?:(?:\\/\\*)(?:(?:[^\\*]+|\\*(?!\\/))*)(?:\\*\\/)))/\n"
  "\n"
  "# --------------------------\n"
  "# Max depth extension\n"
  "# --------------------------\n"
  "# /* max_depth */event inc[] = nulled inc                                                        # Increment depth\n"
  "# /* max_depth */event dec[] = nulled dec                                                        # Decrement depth\n"
  "\n"
  "# ----------------\n"
  "# Number extension\n"
  "# ----------------\n"
  "#\n"
  "# number ::= /\\-?(?:(?:[1-9]?[0-9]+)|[0-9])(?:\\.[0-9]+)?(?:[eE](?:[+-])?[0-9]+)?/ # /* bignum */action => ::lua->lua_number\n"
  "\n"
  "# /* nan */number   ::= '-NaN':i                               action => ::lua->lua_nan\n"
  "# /* nan */number   ::=  'NaN':i                               action => ::lua->lua_nan\n"
  "# /* nan */number   ::= '+NaN':i                               action => ::lua->lua_nan\n"
  "# /* inf */number   ::= '-Infinity':i                          action => ::lua->lua_negative_infinity\n"
  "# /* inf */number   ::=  'Infinity':i                          action => ::lua->lua_positive_infinity\n"
  "# /* inf */number   ::= '+Infinity':i                          action => ::lua->lua_positive_infinity\n"
  "# /* inf */number   ::= '-Inf':i                               action => ::lua->lua_negative_infinity\n"
  "# /* inf            ::=  'Inf':i                               action => ::lua->lua_positive_infinity\n"
  "# /* inf */number   ::= '+Inf':i                               action => ::lua->lua_positive_infinity\n"
  "\n"
  "# -----------------\n"
  "# Control character\n"
  "# -----------------\n"
  "# /* cntrl */char      ::= /[\\x00-\\x1F]/                                                          # Because [:cntrl:] includes DEL (x7F)\n"
  "\n"
  "# -----------------\n"
  "# Lua actions      \n"
  "# -----------------\n"
  "<luascript>\n"
  "  function lua_empty_string()\n"
  "    return ''\n"
  "  end\n"
  "  function lua_unicode(u)\n"
  "    local _hex = {}\n"
  "    local _length = string.len(u)\n"
  "    -- Per def u is a sequence of \\\\u[[:xdigit:]]{4} i.e. 6 'characters', ahem bytes\n"
  "    while (_length > 0) do\n"
  "      local _codepoint = string.sub(u, _length + 3, _length + 6) -- Extract the [[:xdigit:]]{4} part\n"
  "      _length = _length - 6\n"
  "    end\n"
  "    return x\n"
  "  end\n"
  "  function lua_do_empty_array(context)\n"
  "    return {}\n"
  "  end\n"
  "  function lua_do_true(context, x, y)\n"
  "    return true\n"
  "  end\n"
  "  function lua_do_false(context, x, y)\n"
  "    return false\n"
  "  end\n"
  "  function lua_do_number(context, number_as_string)\n"
  "    local value = tonumber(number_as_string)\n"
  "    -- error(number_as_string..'=>'..tostring(value))\n"
  "    return value\n"<commit_msg>test/jsonTesterLua.c: lua_unicode action t+ [ci skip]<commit_after>#include <stdlib.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <genericLogger.h>
#include <marpaESLIF.h>

static short inputReaderb(void *userDatavp, char **inputsp, size_t *inputlp, short *eofbp, short *characterStreambp, char **encodingsp, size_t *encodinglp);

typedef struct marpaESLIFTester_context {
  genericLogger_t *genericLoggerp;
  char            *inputs;
  size_t           inputl;
} marpaESLIFTester_context_t;

const static char *dsl = "\n"
  "#\n"
  "# Default action is to propagate the first RHS value\n"
  "#\n"
  ":default ::= action => ::shift\n"
  "\n"
  "                   #######################################################\n"
  "                   # >>>>>>>>>>>>>>>> Strict JSON Grammar <<<<<<<<<<<<<<<<\n"
  "                   #######################################################\n"
  "\n"
  "# ----------------\n"
  "# Start is a value\n"
  "# ----------------\n"
  ":start ::= value\n"
  "\n"
  "# -------------------\n"
  "# Composite separator\n"
  "# -------------------\n"
  "comma    ::= ','                                  action         => ::undef   # No-op anyway, override ::shift (default action)\n"
  "\n"
  "# ----------\n"
  "# JSON value\n"
  "# ----------\n"
  "value    ::= string                                                           # ::shift (default action)\n"
  "           | number                                                           # ::shift (default action)\n"
  "           | object                                                           # ::shift (default action)\n"
  "           | array                                                            # ::shift (default action)\n"
  "           | 'true'                               action         => ::lua->lua_true      # Returns a lua true value\n"
  "           | 'false'                              action         => ::lua->lua_false     # Returns a lua false value\n"
  "           | 'null'\n"
  "\n"
  "# -----------\n"
  "# JSON object\n"
  "# -----------\n"
  "object   ::= '{' inc members '}' dec              action         => ::copy[2] # Returns members\n"
  "members  ::= pairs*                               action         => ::lua->lua_members   # Returns { @{pairs1}, ..., @{pair2} }\n"
  "                                                  separator      => comma     # ... separated by comma\n"
  "                                                  proper         => 1         # ... with no trailing separator\n"
  "                                                  hide-separator => 1         # ... and hide separator in the action\n"
  "                                                  \n"
  "pairs    ::= string ':' value                     action         => ::lua->lua_pairs     # Returns [ string, value ]\n"
  "\n"
  "# -----------\n"
  "# JSON Arrays\n"
  "# -----------\n"
  "array    ::= '[' inc elements ']' dec             action         => ::copy[2] # Returns elements\n"
  "elements ::= value*                               action => ::lua->lua_elements          # Returns [ value1, ..., valuen ]\n"
  "                                                  separator      => comma     # ... separated by comma\n"
  "                                                  proper         => 1         # ... with no trailing separator\n"
  "                                                  hide-separator => 1         # ... and hide separator in the action\n"
  "                                                  \n"
  "\n"
  "# ------------\n"
  "# JSON Numbers\n"
  "# ------------\n"
  "number ::= NUMBER            # /* bignum */       action => ::lua->lua_number            # Prepare for eventual bignum extension\n"
  "\n"
  "NUMBER   ~ _INT\n"
  "         | _INT _FRAC\n"
  "         | _INT _EXP\n"
  "         | _INT _FRAC _EXP\n"
  "_INT     ~ _DIGIT\n"
  "         | _DIGIT19 _DIGITS\n"
  "         | '-' _DIGIT\n"
  "         | '-' _DIGIT19 _DIGITS\n"
  "_DIGIT   ~ [0-9]\n"
  "_DIGIT19 ~ [1-9]\n"
  "_FRAC    ~ '.' _DIGITS\n"
  "_EXP     ~ _E _DIGITS\n"
  "_DIGITS  ~ _DIGIT+\n"
  "_E       ~ /e[+-]?/i\n"
  "\n"
  "# -----------\n"
  "# JSON String\n"
  "# -----------\n"
  "string     ::= '\"' discardOff chars '\"' discardOn action => ::copy[2]               # Only chars is of interest\n"
  "discardOff ::=                                    action => ::undef                 # Nullable rule used to disable discard\n"
  "discardOn  ::=                                    action => ::undef                 # Nullable rule used to enable discard\n"
  "\n"
  "event :discard[on]  = nulled discardOn                                                           # Implementation of discard disabing using reserved ':discard[on]' keyword\n"
  "event :discard[off] = nulled discardOff                                                          # Implementation of discard enabling using reserved ':discard[off]' keyword\n"
  "\n"
  "chars   ::= filled                                                                               # ::shift (default action)\n"
  "filled  ::= char+                                 action => ::concat                # Returns join('', char1, ..., charn)\n"
  "chars   ::=                                       action => ::lua->lua_empty_string            # Prefering empty string instead of undef\n"
  "char    ::= [^\"\\\\\\x00-\\x1F]                                                         # ::shift (default action) - take care PCRE2 [:cntrl:] includes DEL character\n"
  "          | '\\\\' '\"'                              action => ::copy[1]               # Returns double quote, already ok in data\n"
  "          | '\\\\' '\\\\'                             action => ::copy[1]               # Returns backslash, already ok in data\n"
  "          | '\\\\' '/'                              action => ::copy[1]               # Returns slash, already ok in data\n"
  "          | '\\\\' 'b'                              action => ::u8\"\\x{08}\"\n"
  "          | '\\\\' 'f'                              action => ::u8\"\\x{0C}\"\n"
  "          | '\\\\' 'n'                              action => ::u8\"\\x{0A}\"\n"
  "          | '\\\\' 'r'                              action => ::u8\"\\x{0D}\"\n"
  "          | '\\\\' 't'                              action => ::u8\"\\x{09}\"\n"
  "          | /(?:\\\\u[[:xdigit:]]{4})+/             action => ::lua->lua_unicode\n"
  "\n"
  "\n"
  "# -------------------------\n"
  "# Unsignificant whitespaces\n"
  "# -------------------------\n"
  ":discard ::= /[\\x{9}\\x{A}\\x{D}\\x{20}]+/\n"
  "\n"
  "# ------------------------------------------------------\n"
  "# Needed for eventual depth extension - no op by default\n"
  "# ------------------------------------------------------\n"
  "inc ::=                                                        action => ::undef\n"
  "dec ::=                                                        action => ::undef\n"
  "\n"
  "                   #######################################################\n"
  "                   # >>>>>>>>>>>>>>>>>> JSON Extensions <<<<<<<<<<<<<<<<<<\n"
  "                   #######################################################\n"
  "\n"
  "# --------------------------\n"
  "# Unlimited commas extension\n"
  "# --------------------------\n"
  "# /* Unlimited commas */commas   ::= comma+\n"
  "\n"
  "# --------------------------\n"
  "# Perl comment extension\n"
  "# --------------------------\n"
  "# /* Perl comment */:discard ::= /(?:(?:#)(?:[^\\n]*)(?:\\n|\\z))/u\n"
  "\n"
  "# --------------------------\n"
  "# C++ comment extension\n"
  "# --------------------------\n"
  "# /* C++ comment */:discard ::= /(?:(?:(?:\\/\\/)(?:[^\\n]*)(?:\\n|\\z))|(?:(?:\\/\\*)(?:(?:[^\\*]+|\\*(?!\\/))*)(?:\\*\\/)))/\n"
  "\n"
  "# --------------------------\n"
  "# Max depth extension\n"
  "# --------------------------\n"
  "# /* max_depth */event inc[] = nulled inc                                                        # Increment depth\n"
  "# /* max_depth */event dec[] = nulled dec                                                        # Decrement depth\n"
  "\n"
  "# ----------------\n"
  "# Number extension\n"
  "# ----------------\n"
  "#\n"
  "# number ::= /\\-?(?:(?:[1-9]?[0-9]+)|[0-9])(?:\\.[0-9]+)?(?:[eE](?:[+-])?[0-9]+)?/ # /* bignum */action => ::lua->lua_number\n"
  "\n"
  "# /* nan */number   ::= '-NaN':i                               action => ::lua->lua_nan\n"
  "# /* nan */number   ::=  'NaN':i                               action => ::lua->lua_nan\n"
  "# /* nan */number   ::= '+NaN':i                               action => ::lua->lua_nan\n"
  "# /* inf */number   ::= '-Infinity':i                          action => ::lua->lua_negative_infinity\n"
  "# /* inf */number   ::=  'Infinity':i                          action => ::lua->lua_positive_infinity\n"
  "# /* inf */number   ::= '+Infinity':i                          action => ::lua->lua_positive_infinity\n"
  "# /* inf */number   ::= '-Inf':i                               action => ::lua->lua_negative_infinity\n"
  "# /* inf            ::=  'Inf':i                               action => ::lua->lua_positive_infinity\n"
  "# /* inf */number   ::= '+Inf':i                               action => ::lua->lua_positive_infinity\n"
  "\n"
  "# -----------------\n"
  "# Control character\n"
  "# -----------------\n"
  "# /* cntrl */char      ::= /[\\x00-\\x1F]/                                                          # Because [:cntrl:] includes DEL (x7F)\n"
  "\n"
  "# -----------------\n"
  "# Lua actions      \n"
  "# -----------------\n"
  "<luascript>\n"
  "  function lua_empty_string()\n"
  "    return ''\n"
  "  end\n"
  "  function lua_unicode(u)\n"
  "    local _hex = {}\n"
  "    local _length = string.len(u)\n"
  "    local _nextArrayIndice = 1\n"
  "\n"
  "    -- Per def u is a sequence of \\\\u[[:xdigit:]]{4} i.e. 6 'characters', ahem bytes\n"
  "    while (_length > 0) do\n"
  "      local _codepointAsString = string.sub(u, _length + 3, _length + 6) -- Extract the [[:xdigit:]]{4} part\n"
  "      local _codepoint = tonumber(_codepointAsString, 16)\n"
  "      _hex[_nextArrayIndice] = _codepoint\n"
  "      _nextArrayIndice = _nextArrayIndice + 1\n"
  "\n"
  "      _length = _length - 6\n"
  "    end\n"
  "    return x\n"
  "  end\n"
  "  function lua_do_empty_array(context)\n"
  "    return {}\n"
  "  end\n"
  "  function lua_do_true(context, x, y)\n"
  "    return true\n"
  "  end\n"
  "  function lua_do_false(context, x, y)\n"
  "    return false\n"
  "  end\n"
  "  function lua_do_number(context, number_as_string)\n"
  "    local value = tonumber(number_as_string)\n"
  "    -- error(number_as_string..'=>'..tostring(value))\n"
  "    return value\n"
{'fraction_non_alphanumeric': 0.256181673157228, 'fraction_numerical': 0.006287242528034533, 'mean_word_length': 1.0390318568831913, 'pattern_counts': {'":': 6, '<': 85, '<?xml version=': 0, '>': 209, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 114, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3908606', 'n_tokens_mistral': 6366, 'n_tokens_neox': 6236, 'n_words': 1284}