<commit_before>/* Reworked for grammar specificity by <NAME>. Did a bunch of
   work so that rather than reading "a bunch of crap in parens" some
   syntactic information is preserved and recovered. Dec. 14 2014.

   Converted to ANTLR 4 by <NAME>. Unsure of provence. I see
   it commited by matthias.koester for clojure-eclipse project on
   Oct 5, 2009:

   https://code.google.com/p/clojure-eclipse/

   Seems to me Laurent Petit had a version of this. I also see
   Jingguo Yao submitting a link to a now-dead github project on
   Jan 1, 2011.

   https://github.com/laurentpetit/ccw/tree/master/clojure-antlr-grammar

   Regardless, there are some issues perhaps related to "sugar";
   I've tried to fix them.

   This parses https://github.com/weavejester/compojure project.

   I also note this is hardly a grammar; more like "match a bunch of
   crap in parens" but I guess that is LISP for you ;)
 */

grammar Clojure;

file: form *;

form: literal
    | list
    | vector
    | map
    | reader_macro
    ;

forms: form* ;

list: '(' forms ')' ;

vector: '[' forms ']' ;

map: '{' (form form)* '}' ;

set: '#{' forms '}' ;

reader_macro
    : lambda
    | meta_data
    | regex
    | var_quote
    | host_expr
    | set
    | tag
    | discard
    | dispatch
    | deref
    | quote
    | backtick
    | unquote
    | unquote_splicing
    | gensym
    ;

// TJP added '&' (gather a variable number of arguments)
quote
    : '\'' form
    ;

backtick
    : '`' form
    ;

unquote
    : '~' form
    ;

unquote_splicing
    : '~@' form
    ;

tag
    : '^' form form
    ;

deref
    : '@' form
    ;

gensym
    : SYMBOL '#'
    ;

lambda
    : '#(' form* ')'
    ;

meta_data
    : '#^' (map form | form) 
    ;

var_quote
    : '#\'' symbol
    ;

host_expr
    : '#+' form form
    ;

discard
    : '#_' form
    ;

dispatch
    : '#' symbol form
    ;

regex
    : '#' string
    ;

literal
    : string
    | number
    | character
    | nil
    | boolean
    | keyword
    | symbol
    | param_name
    ;

string: STRING;
float: FLOAT;
hex: HEX;
bin: BIN;
bign: BIGN;
long: LONG;
number
    : float
    | hex
    | bin
    | bign
    | long
    ;

character
    : named_char
    | u_hex_quad
    | any_char
    ;
named_char: CHAR_NAMED ;
any_char: CHAR_ANY ;
u_hex_quad: CHAR_U ;

nil: NIL;
boolean: BOOLEAN;

keyword: macro_keyword | simple_keyword;
simple_keyword: ':' symbol;
macro_keyword: ':' ':' symbol;

symbol: ns_symbol | simple_sym;<commit_msg>rm keywords from Clojure. Fixes #236<commit_after>/* Reworked for grammar specificity by <NAME>. Did a bunch of
   work so that rather than reading "a bunch of crap in parens" some
   syntactic information is preserved and recovered. Dec. 14 2014.

   Converted to ANTLR 4 by <NAME>. Unsure of provence. I see
   it commited by matthias.koester for clojure-eclipse project on
   Oct 5, 2009:

   https://code.google.com/p/clojure-eclipse/

   Seems to me Laurent Petit had a version of this. I also see
   <NAME> submitting a link to a now-dead github project on
   Jan 1, 2011.

   https://github.com/laurentpetit/ccw/tree/master/clojure-antlr-grammar

   Regardless, there are some issues perhaps related to "sugar";
   I've tried to fix them.

   This parses https://github.com/weavejester/compojure project.

   I also note this is hardly a grammar; more like "match a bunch of
   crap in parens" but I guess that is LISP for you ;)
 */

grammar Clojure;

file: form *;

form: literal
    | list
    | vector
    | map
    | reader_macro
    ;

forms: form* ;

list: '(' forms ')' ;

vector: '[' forms ']' ;

map: '{' (form form)* '}' ;

set: '#{' forms '}' ;

reader_macro
    : lambda
    | meta_data
    | regex
    | var_quote
    | host_expr
    | set
    | tag
    | discard
    | dispatch
    | deref
    | quote
    | backtick
    | unquote
    | unquote_splicing
    | gensym
    ;

// TJP added '&' (gather a variable number of arguments)
quote
    : '\'' form
    ;

backtick
    : '`' form
    ;

unquote
    : '~' form
    ;

unquote_splicing
    : '~@' form
    ;

tag
    : '^' form form
    ;

deref
    : '@' form
    ;

gensym
    : SYMBOL '#'
    ;

lambda
    : '#(' form* ')'
    ;

meta_data
    : '#^' (map form | form) 
    ;

var_quote
    : '#\'' symbol
    ;

host_expr
    : '#+' form form
    ;

discard
    : '#_' form
    ;

dispatch
    : '#' symbol form
    ;

regex
    : '#' string
    ;

literal
    : string
    | number
    | character
    | nil
    | BOOLEAN
    | keyword
    | symbol
    | param_name
    ;

string: STRING;
hex: HEX;
bin: BIN;
bign: BIGN;
number
    : FLOAT
    | hex
    | bin
    | bign
    | LONG
    ;

character
    : named_char
    | u_hex_quad
    | any_char
    ;
named_char: CHAR_NAMED ;
any_char: CHAR_ANY ;
u_hex_quad: CHAR_U ;

nil: NIL;

keyword: macro_keyword | simple_keyword;
simple_keyword: ':' symbol;
macro_keyword: ':' ':' symbol;

symbol: ns_symbol | simple_sym;
{'fraction_non_alphanumeric': 0.11391375101708706, 'fraction_numerical': 0.007526444263628967, 'mean_word_length': 2.2263779527559056, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 4, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1632333', 'n_tokens_mistral': 1874, 'n_tokens_neox': 1785, 'n_words': 509}