<commit_before># Smaragdine Grammar
This is a work in progress.   
The grammar is described using a modified Wirth syntax notation.

Table of symbols:

| Usage | Notation |
| ----- | -------- |
| definition | `=` |
| alternation | `|` |
| zero or one | `[ ... ]`|
| zero or more | `{ ... }` |
| grouping | `( ... )` |
| comment | `(* ... *)` |
| terminal string | `' ... '` |
| terminal string | `" ... "` |

```ebnf
(* basics *)

digit              = '0'...'9'
char               = (* Any UTF-8 code-point *)
char_ascii         = 'a'...'z'
                   | 'A'...'Z'

identifier         = char_ascii { char_ascii | digit | '_' | '!' | '?' }

(* literals *)
literal_integer    = [ '0' ( 'x' | 'b' ) ] { digit }
literal_float      = ( { digit } '.' digit { digit } )
                   | ( '.' digit { digit } )
literal_char       = "'" char "'"
literal_string     = '"' [ { char } ] '"'
literal_raw_string = 'r' literal_string
literal            = literal_char
                   | literal_string
                   | literal_raw_string
                   | literal_integer

(* operators *)

op_unary           = '+'
                   | '-'
                   | '~'
                   | '!'
op_assignment      = '='
                   | '&='
                   | '|='
                   | '^='
                   | '+='
                   | '-='
                   | '*='
                   | '/='
                   | '%='
                   | '<<='
                   | '>>='<commit_msg>fix and shorten like 2 things<commit_after># Smaragdine Grammar
This is a work in progress.   
The grammar is described using a modified Wirth syntax notation.

Table of symbols:

| Usage | Notation |
| ----- | -------- |
| definition | `=` |
| alternation | `|` |
| zero or one | `[ ... ]`|
| zero or more | `{ ... }` |
| grouping | `( ... )` |
| comment | `(* ... *)` |
| terminal string | `' ... '` |
| terminal string | `" ... "` |

```ebnf
(* basics *)

digit              = '0'...'9'
char               = (* Any UTF-8 code-point *)
char_ascii         = 'a'...'z'
                   | 'A'...'Z'

identifier         = char_ascii { char_ascii | digit | '_' | '!' | '?' }

(* literals *)
literal_integer    = [ '0' ( 'x' | 'b' ) ] digit { digit }
literal_float      = ( { digit } '.' digit { digit } )
                   | ( '.' digit { digit } )
literal_char       = "'" char "'"
literal_string     = '"' { char } '"'
literal_raw_string = 'r' literal_string
literal            = literal_char
                   | literal_string
                   | literal_raw_string
                   | literal_integer

(* operators *)

op_unary           = '+'
                   | '-'
                   | '~'
                   | '!'
op_assignment      = '='
                   | '&='
                   | '|='
                   | '^='
                   | '+='
                   | '-='
                   | '*='
                   | '/='
                   | '%='
                   | '<<='
                   | '>>='
{'fraction_non_alphanumeric': 0.19228210246174318, 'fraction_numerical': 0.0029940119760479044, 'mean_word_length': 1.116115411681914, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2786426', 'n_tokens_mistral': 943, 'n_tokens_neox': 874, 'n_words': 166}