<commit_before>grammar ConfigurationFile;

configuration
    :    setting_list
    ;

setting_list
    :    setting+
    ;

setting
    :    NAME (':' | '=') value (';' | ',')?
    ;

value
    :    scalar_value | array | list | group
    ;

value_list
    :    value (',' value)*
    ;

scalar_value
    :    BOOLEAN
    |    INTEGER
    |    INTEGER64
    |    HEX
    |    HEX64
    |    FLOAT
    |    STRING+
    ;

scalar_value_list
    :    scalar_value (',' scalar_value)*
    ;

array
    :    '[' scalar_value_list? ']'
    ;

list
    :    '(' value_list? ')'
    ;

group
    :    '{' setting_list? '}'
    ;

BOOLEAN
    :    ('T'|'t')('R'|'r')('U'|'u')('E'|'e')
    |    ('F'|'f')('A'|'a')('L'|'l')('S'|'s')('E'|'e')
    ;

fragment ESCAPE_SEQUENCE
    :    '\\' ('\\' | '"' | 't' | 'n' | 'r')
    ;

STRING
    :    '"' (ESCAPE_SEQUENCE | ~('"' | '\\'))* '"'
    ;

NAME
    :    ('A'..'Z'|'a'..'z'|'*') ('-'|'A'..'Z'|'a'..'z'|'0'..'9'|'_'|'*')*
    ;

fragment DIGITS
    :    ('-'|'+')? ('0'..'9')+
    ;

INTEGER
    :    DIGITS
    ;

INTEGER64
    :    DIGITS 'L' 'L'?
    ;

fragment HEXDIGITS
    :    '0' ('X'|'x') ('0'..'9'|'A'..'F'|'a'..'f')+
    ;

HEX
    :    HEXDIGITS
    ;

HEX64
    :    HEXDIGITS 'L' 'L'?
    ;

FLOAT
    :     ('-'|'+')? ('0'..'9')* '.' ('0'..'9')* (('e'|'E') ('-'|'+')? ('0'..'9')+)?
    ;

/* Ignore white space */
WS
    :    (' '|'\t'|'\f'|'\n'|'\r')+ { skip(); }
    ;

ML_COMMENT
    :    '/*' .* '*/' { skip(); }
    ;

OL_COMMENT
    :    '#' ~( '\r' | '\n' )* { skip(); }
    |    '//' ~( '\r' | '\n' )* { skip(); }
    ;
<commit_msg>Fix float lexer rule<commit_after>grammar ConfigurationFile;

configuration
    :    setting_list
    ;

setting_list
    :    setting+
    ;

setting
    :    NAME (':' | '=') value (';' | ',')?
    ;

value
    :    scalar_value | array | list | group
    ;

value_list
    :    value (',' value)*
    ;

scalar_value
    :    BOOLEAN
    |    INTEGER
    |    INTEGER64
    |    HEX
    |    HEX64
    |    FLOAT
    |    STRING+
    ;

scalar_value_list
    :    scalar_value (',' scalar_value)*
    ;

array
    :    '[' scalar_value_list? ']'
    ;

list
    :    '(' value_list? ')'
    ;

group
    :    '{' setting_list? '}'
    ;

BOOLEAN
    :    ('T'|'t')('R'|'r')('U'|'u')('E'|'e')
    |    ('F'|'f')('A'|'a')('L'|'l')('S'|'s')('E'|'e')
    ;

fragment ESCAPE_SEQUENCE
    :    '\\' ('\\' | '"' | 't' | 'n' | 'r')
    ;

STRING
    :    '"' (ESCAPE_SEQUENCE | ~('"' | '\\'))* '"'
    ;

NAME
    :    ('A'..'Z'|'a'..'z'|'*') ('-'|'A'..'Z'|'a'..'z'|'0'..'9'|'_'|'*')*
    ;

fragment DIGITS
    :    ('-'|'+')? ('0'..'9')+
    ;

INTEGER
    :    DIGITS
    ;

INTEGER64
    :    DIGITS 'L' 'L'?
    ;

fragment HEXDIGITS
    :    '0' ('X'|'x') ('0'..'9'|'A'..'F'|'a'..'f')+
    ;

HEX
    :    HEXDIGITS
    ;

HEX64
    :    HEXDIGITS 'L' 'L'?
    ;

FLOAT
    :     ('-'|'+')? ('0'..'9')* ('.' ('0'..'9')*)? (('e'|'E') ('-'|'+')? ('0'..'9')+)?
    ;

/* Ignore white space */
WS
    :    (' '|'\t'|'\f'|'\n'|'\r')+ { skip(); }
    ;

ML_COMMENT
    :    '/*' .* '*/' { skip(); }
    ;

OL_COMMENT
    :    '#' ~( '\r' | '\n' )* { skip(); }
    |    '//' ~( '\r' | '\n' )* { skip(); }
    ;

{'fraction_non_alphanumeric': 0.3021875, 'fraction_numerical': 0.013125, 'mean_word_length': 1.6854026845637584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3469692', 'n_tokens_mistral': 1575, 'n_tokens_neox': 1529, 'n_words': 177}