<commit_before>class ASTNode { 

  refine() : ASTNode { return this; }

}

class Num extends ASTNode { 

  constructor(public num : number) { super(); }

}

class Symbol extends ASTNode {

  constructor(public symbol : string) { super(); }

}

class List extends ASTNode {

  constructor(public list : Array<ASTNode>) { super(); }

  refine() : ASTNode {
    return null;
  }

}

class Tuple extends ASTNode {

  constructor(public elements : Array<ASTNode>) { super(); }

  refine() : ASTNode {
    return null;
  }

}

class SExpr extends ASTNode {

  constructor(public expressions : Array<ASTNode>) { super(); }

  refine() : ASTNode {
    var head : ASTNode = this.expressions[0];
    return null;
  }


}

// Non-initial nodes that only result from the refinement process.<commit_msg>filling in ast normalization code<commit_after>class ASTNode { 

  refine() : ASTNode { return this; }

  symbol() : boolean { return false; }

}

class Num extends ASTNode { 

  constructor(private num : number) { super(); }

}

class Symbol extends ASTNode {

  constructor(public symb : string) { super(); }

  symbol() : boolean { return true; }

}

class List extends ASTNode {

  constructor(private list : Array<ASTNode>) { super(); }

  refine() : ASTNode {
    return null;
  }

}

class Tuple extends ASTNode {

  constructor(private elements : Array<ASTNode>) { super(); }

  refine() : ASTNode {
    return new Tuple(this.elements.map(x => x.refine()));
  }

}

class SExpr extends ASTNode {

  constructor(private expressions : Array<ASTNode>) { super(); }

  // Arguments must be a list of symbols.
  refine_argument_definitions() : Array<Symbol> {
    return this.expressions.map(x => {
      if (x.symbol()) {
        return (<Symbol>x);
      } else {
        throw new Error("Function argument definitions must be symbols.");
      }
    });
  }

  // a.k.a. normalization
  refine() : ASTNode {
    var exprs : Array<ASTNode> = this.expressions;
    var refined_head : ASTNode = exprs[0].refine();
    if (refined_head.symbol()) {
      // Dealing with a symbol so lets figure out if we need to refine it further
      switch((<Symbol>refined_head).symb) {
        case 'fun': // function definition
          return new AnonymousFunction((<SExpr>exprs[1]).refine_argument_definitions(), exprs[2].refine_definition_body());
        case 'let': // let binding
          return new LetExpressions(exprs[1].refine_let_bindings(), exprs[2].refine_let_body());
        case 'match': // pattern matching
          return new MatchExpression(exprs[1].refine_match_value(), exprs[2].refine_match_patterns());
        case 'if': // conditional
          return new IfExpression(exprs[1].refine_if_condition(), exprs[2].refine_true_branch(), exprs[3].refine_false_branch());
        default:
          throw new Error('Unknown symbol: ' + refined_head.symbol);
      }
    }
    // Otherwise it must be a function call.
    return new FunctionCall((<Symbol>refined_head), exprs.slice(1).map(x => x.refine()));
  }

}

// Non-initial nodes that only result from the refinement process.
class FunctionCall extends ASTNode {

  constructor(private name : Symbol, private args : Array<ASTNode>) { super(); }

}

class AnonymousFunction extends ASTNode {

  constructor(private args : Array<Symbol>, private body : ASTNode) { super(); }

}

{'fraction_non_alphanumeric': 0.1280743851229754, 'fraction_numerical': 0.003599280143971206, 'mean_word_length': 3.303225806451613, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '383732', 'n_tokens_mistral': 1058, 'n_tokens_neox': 991, 'n_words': 273}