<commit_before>DataShape Grammar
=================

The DataShape grammar.

Grammar::

    top : mod

    mod : mod mod
        | stmt

    stmt : TYPE lhs_expression EQUALS rhs_expression
         | rhs_expression

    lhs_expression : lhs_expression lhs_expression
                   | NAME

    rhs_expression : rhs_expression_list

    rhs_expression_list : rhs_expression_list COMMA rhs_expression_list
                   | appl
                   | record
                   | BIT
                   | NAME
                   | NUMBER

    appl_args : appl_args COMMA appl_args
              | appl
              | record
              | '(' rhs_expression ')'
              | BIT
              | NAME
              | NUMBER
              | STRING

    appl : NAME '(' appl_args ')'

    record : LBRACE record_opt RBRACE
    record_opt : record_opt SEMI record_opt
               | record_item
               | empty
    record_name : NAME
                | BIT
                | TYPE
    record_item : record_name COLON '(' rhs_expression ')'
                | record_name COLON rhs_expression'

    empty :<commit_msg>Rewrite the datashape grammar document with a new grammar<commit_after>DataShape Grammar
=================

The datashape language is a DSL which describes the structure of data, abstracted from
a particular implementation in a language or file format. Compared to the Python
library NumPy, it combines `shape` and `dtype` together, and introduces a
syntax for describing structured data.

Some of the basic features include:

* Dimensions are separated by commas.
* Lists of types are separated by semicolons.
* Types and Typevars are distinguished by the capitalization of the leading
  character. Lowercase for types, and uppercase for typevars.
* Type constructors operate using square brackets.

Some Simple Examples
--------------------

Here are some simple examples to motivate the idea::

    # Scalar types
    bool
    int32
    float64

    # Arrays
    3, 4, int32
    10, var, float64
    3, complex[float64]

    # Array of Structures
    100, {
        name: string;
        birthday: date;
        address: {
            street: string;
            city: string;
            postalcode: string;
            country: string;
        };
    }


The DataShape Grammar
---------------------

Dimension Type Symbol Table::

    var : variable dimension

Data Type Symbol Table::

    # Numeric
    bool
    int8
    int16
    int32
    int64
    int128
    uint8
    uint16
    uint32
    uint64
    uint128
    float16
    float32
    float64
    float128

    # Other
    string
    bytes
    date
    json

Data Type Constructor Symbol Table::

    # complex[float32], complex[float64]
    complex
    # string[ascii], string[cp949]
    string
    # bytes[4;2]
    bytes
    # datetime[minutes;CST]
    datetime
    # categorical[string; ['low', 'medium', 'high']]
    categorical

Tokens::

    NAME_LOWER : [a-z][a-zA-Z0-9_]*
    NAME_UPPER : [A-Z][a-zA-Z0-9_]*
    NAME_OTHER : _[a-zA-Z0-9_]*
    COMMA : ,
    SEMI : ;
    ELLIPSIS : \.\.\.
    LBRACKET : \[
    RBRACKET : \]
    LBRACE : \{
    RBRACE : \}
    INTEGER : 0|[1-9][0-9]*
    STRING : (?:"(?:[^"\n\r\\]|(?:\\x[0-9a-fA-F]{2})|(?:\\u[0-9a-fA-F]{4})|(?:\\.))*")|(?:\'(?:[^\'\n\r\\]|(?:\\x[0-9a-fA-F]+)|(?:\\u[0-9a-fA-F]{4})|(?:\\.))*\')


Grammar::

    # Comma-separated list of dimensions, followed by data type
    datashape : dim COMMA datashape
              | dtype

    # Dimension Type (from the dimension type symbol table)
    dim : symbol_type
        | ellipsis_typevar
        | INTEGER


    # Data Type (from the data type symbol table)
    dtype : symbol_type
          | struct_type

    # A type defined by a symbol
    symbol_type : typevar
                | type
                | type_constr

    # A type variable
    typevar : NAME_UPPER

    # A type variable with ellipsis
    ellipsis_typevar : NAME_UPPER ELLIPSIS

    # A bare type (from the data type symbol table)
    type : NAME_LOWER

    # Type Constructor (from the data type constructor symbol table)
    type_constr : NAME_LOWER LBRACKET type_arg_list RBRACKET

    # Type Constructor: list of arguments
    type_arg_list : type_arg COMMA type_arg_list
                  | type_arg

    # Type Constructor : single argument
    type_arg : datashape
             | INTEGER
             | STRING
             | list_type_arg

    # Type Constructor : single list argument
    list_type_arg : empty_list
                  | LBRACKET datashape_list RBRACKET
                  | LBRACKET integer_list RBRACKET
                  | LBRACKET string_list RBRACKET

    empty_list : LBRACKET RBRACKET

    datashape_list : datashape SEMI datashape_list
                   | datashape

    integer_list : INTEGER SEMI integer_list
                 | INTEGER

    string_list : STRING SEMI string_list
                | STRING


    # Struct/Record type
    struct_type : LBRACE struct_field_list RBRACE

    struct_field_list : struct_field SEMI struct_field_list
                      | struct_field

    struct_field : struct_field_name COLON datashape

    struct_field_name : NAME_LOWER
                      | NAME_UPPER
                      | NAME_OTHER
    

{'fraction_non_alphanumeric': 0.08973137740522004, 'fraction_numerical': 0.014859973328253, 'mean_word_length': 1.8735632183908046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1743606', 'n_tokens_mistral': 1678, 'n_tokens_neox': 1547, 'n_words': 438}