<commit_before>tree grammar TreeWalker;

options {
  tokenVocab = Grammar;
  output = AST;
  ASTLabelType = CommonTree;
}

@header {
  package interpres;
  import interpres.ast.*;
  import interpres.ast.Node;
}

walk returns [Node node]: program { $node = $program.node; };

// Transformer rules
// =============================================================================

program returns [Node node]
  @init {
    List<Node> expressions = new ArrayList<Node>();
  }
  @after {
    $node = new ListNode(expressions);
  }
  :
  ^(LIST (expression { expressions.add($expression.node); })*)
  ;

expression returns [Node node]: sexp | list | atom;

sexp returns [Node node]
  @init {
    Node functionName;
    List<Node> arguments = new ArrayList<Node>();
  }
  @after {
    $node = new SexpNode(functionName, arguments);
  }
  :
  ^(SEXPR
    sexp_function_name { functionName = $sexp_function_name.node; }
    (sexp_argument { arguments.add($sexp_argument.node); })*)
  ;

sexp_function_name returns [Node node]: expression;

sexp_argument returns [Node node]: expression;

list returns [Node node]
  @init {
    List<Node> items = new ArrayList<Node>();
  }
  @after {
    $node = new ListNode(items);
  }
  :
  ^(LIST (list_item { items.add($list_item.node); })*)
  ;

list_item returns [Node node]: expression;

atom returns [Node node]:
  ^(STRING string) { $node = new StringNode($string.text); }
  ^(REFERENCE reference) { $node = new ReferenceNode($reference.text); }
  ;

string: QUOTED_VALUE;
reference: IDENTIFIER;

<commit_msg>Renamed Node to AST<commit_after>tree grammar TreeWalker;

options {
  tokenVocab = Grammar;
  output = AST;
  ASTLabelType = CommonTree;
}

@header {
  package interpres;
  import interpres.ast.*;
}

walk returns [AST ast]: program { $ast = $program.ast; };

// Transformer rules
// =============================================================================

program returns [AST ast]
  @init {
    List<AST> expressions = new ArrayList<AST>();
  }
  @after {
    $ast = new ListExpression(expressions);
  }
  :
  ^(LIST (expression { expressions.add($expression.ast); })*)
  ;

expression returns [AST ast]
  : sexp { $ast = $sexp.ast; }
  | list { $ast = $list.ast; }
  | atom { $ast = $atom.ast; }
  ;

sexp returns [AST ast]
  @init {
    AST functionName;
    List<AST> arguments = new ArrayList<AST>();
  }
  @after {
    $ast = new SymbolicExpression(functionName, arguments);
  }
  :
  ^(SEXPR
    sexp_function_name { functionName = $sexp_function_name.ast; }
    (sexp_argument { arguments.add($sexp_argument.ast); })*)
  ;

sexp_function_name returns [AST ast]: expression { $ast = $expression.ast; };

sexp_argument returns [AST ast]: expression { $ast = $expression.ast; };

list returns [AST ast]
  @init {
    List<AST> items = new ArrayList<AST>();
  }
  @after {
    $ast = new ListExpression(items);
  }
  :
  ^(LIST (list_item { items.add($list_item.ast); })*)
  ;

list_item returns [AST ast]: expression { $ast = $expression.ast; };

atom returns [AST ast]
  : ^(STRING string=QUOTED_VALUE) {
    String literal = $string.text;
    $ast = new StringLiteral(literal.substring(1, literal.length() - 1));
  }
  | ^(REFERENCE reference=IDENTIFIER) {
    $ast = new Reference($reference.text);
  }
  ;


{'fraction_non_alphanumeric': 0.1886099203919167, 'fraction_numerical': 0.000612369871402327, 'mean_word_length': 3.8543833580980684, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6786475', 'n_tokens_mistral': 1097, 'n_tokens_neox': 1073, 'n_words': 254}