<commit_before>package water.exec;

import java.util.*;
import water.*;
import water.fvec.*;

/** Parse & execute a generic R-like string, in the context of an H2O Cloud
 *  @author <EMAIL>
 */
public class Exec2 {
  // Parse a string, execute it & return a Frame.
  // Basic types: Frame, Scalar, Function
  // Functions are 1st class; every argument typed one of the above.
  // Assignment is always to in-scope variables only.
  // Initial environment has all Frame Keys mapped to Frame-typed variables

  // Big Allocation: all expressions are eval'd in a context where a large temp
  // is available, and all allocations are compatible with that temp.  Linear-
  // logic style execution is guaranteed inside the big temp.  Parse error if
  // an expression which is not provably small does not have an active temp.

  // Grammar:
  //   statements := cxexpr ; statements
  //   cxexpr :=                   // COMPLEX expr 
  //           iexpr               // Simple RHS-expr
  //           id = cxexpr         // Shadows outer var with a ptr assignment; no copy, read-only
  //                               // Overwrites inner var; types must match.
  //           id := cxexpr        // Deep-copy writable temp; otherwise same as above
  //           id[] = cxexpr       // Only for writable temps; slice assignment
  //           id[] := cxexpr      // Deep-copy, then slice assignment
  //           iexpr ? cxexpr : cxexpr  // exprs must have equal types
  //   iexpr := 
  //           slice {op2 slice}*  // Infix notation, evals LEFT TO RIGHT
  //   slice := 
  //           expr                // Can be a dbl or fcn or ary
  //           expr[]              // whole ary val
  //           expr[,]             // whole ary val
  //           expr[cxexpr,cxexpr] // row & col ary slice (row FIRST, col SECOND)
  //           expr[,cxexpr]       // col-only ary slice
  //           expr[cxexpr,]       // row-only ary slice
  //   expr :=
  //           val
  //           val(cxexpr,...)*    // Prefix function application, evals LEFT TO RIGHT
  //   val :=
  //           ( cxexpr )          // Ordering evaluation
  //           id                  // any visible var; will be typed
  //           num                 // Scalars, treated as 1x1
  //           op                  // Built-in functions
  //           function(v0,v1,v2) { statements; ...v0,v1,v2... } // 1st-class lexically scoped functions
  //   op  := sgn sin cos ...any unary op...
  //   op  := min max + - * / % & |    ...any boolean op...
  //   op  := c // R's "c" operator, returns a Nx1 array
  //   op  := ary byCol(ary,dbl op2(dbl,dbl))

  public static Env exec( String str ) throws IllegalArgumentException {
    // Preload the global environment from existing Frames
    Env env = new Env();
    ArrayList<ASTId> global = new ArrayList<ASTId>();
    for( Value v : H2O.values() )
      if( v.type()==TypeMap.FRAME ) { // Add to parser's namespace
        global.add(new ASTId(Type.ARY,v._key.toString(),0,global.size()));
        env.push((Frame)v.get(),v._key.toString());
      }

    // Some global constants
    global.add(new ASTId(Type.DBL,"T",0,global.size()));  env.push(1.0);
    global.add(new ASTId(Type.DBL,"F",0,global.size()));  env.push(0.0);

    // Parse.  Type-errors get caught here and throw IAE
    int argcnt = global.size();
    AST ast = new Exec2(str,global).parse();

    try {
      env.push(global.size()-argcnt);   // Push space for temps<commit_msg>Add NA to Exec2<commit_after>package water.exec;

import java.util.*;
import water.*;
import water.fvec.*;

/** Parse & execute a generic R-like string, in the context of an H2O Cloud
 *  @author <EMAIL>
 */
public class Exec2 {
  // Parse a string, execute it & return a Frame.
  // Basic types: Frame, Scalar, Function
  // Functions are 1st class; every argument typed one of the above.
  // Assignment is always to in-scope variables only.
  // Initial environment has all Frame Keys mapped to Frame-typed variables

  // Big Allocation: all expressions are eval'd in a context where a large temp
  // is available, and all allocations are compatible with that temp.  Linear-
  // logic style execution is guaranteed inside the big temp.  Parse error if
  // an expression which is not provably small does not have an active temp.

  // Grammar:
  //   statements := cxexpr ; statements
  //   cxexpr :=                   // COMPLEX expr 
  //           iexpr               // Simple RHS-expr
  //           id = cxexpr         // Shadows outer var with a ptr assignment; no copy, read-only
  //                               // Overwrites inner var; types must match.
  //           id := cxexpr        // Deep-copy writable temp; otherwise same as above
  //           id[] = cxexpr       // Only for writable temps; slice assignment
  //           id[] := cxexpr      // Deep-copy, then slice assignment
  //           iexpr ? cxexpr : cxexpr  // exprs must have equal types
  //   iexpr := 
  //           slice {op2 slice}*  // Infix notation, evals LEFT TO RIGHT
  //   slice := 
  //           expr                // Can be a dbl or fcn or ary
  //           expr[]              // whole ary val
  //           expr[,]             // whole ary val
  //           expr[cxexpr,cxexpr] // row & col ary slice (row FIRST, col SECOND)
  //           expr[,cxexpr]       // col-only ary slice
  //           expr[cxexpr,]       // row-only ary slice
  //   expr :=
  //           val
  //           val(cxexpr,...)*    // Prefix function application, evals LEFT TO RIGHT
  //   val :=
  //           ( cxexpr )          // Ordering evaluation
  //           id                  // any visible var; will be typed
  //           num                 // Scalars, treated as 1x1
  //           op                  // Built-in functions
  //           function(v0,v1,v2) { statements; ...v0,v1,v2... } // 1st-class lexically scoped functions
  //   op  := sgn sin cos ...any unary op...
  //   op  := min max + - * / % & |    ...any boolean op...
  //   op  := c // R's "c" operator, returns a Nx1 array
  //   op  := ary byCol(ary,dbl op2(dbl,dbl))

  public static Env exec( String str ) throws IllegalArgumentException {
    // Preload the global environment from existing Frames
    Env env = new Env();
    ArrayList<ASTId> global = new ArrayList<ASTId>();
    for( Value v : H2O.values() )
      if( v.type()==TypeMap.FRAME ) { // Add to parser's namespace
        global.add(new ASTId(Type.ARY,v._key.toString(),0,global.size()));
        env.push((Frame)v.get(),v._key.toString());
      }

    // Some global constants
    global.add(new ASTId(Type.DBL,"T",0,global.size()));  env.push(1.0);
    global.add(new ASTId(Type.DBL,"F",0,global.size()));  env.push(0.0);
    global.add(new ASTId(Type.DBL,"NA",0,global.size()));  env.push(Double.NaN);

    // Parse.  Type-errors get caught here and throw IAE
    int argcnt = global.size();
    AST ast = new Exec2(str,global).parse();

    try {
      env.push(global.size()-argcnt);   // Push space for temps
{'fraction_non_alphanumeric': 0.12693454493823655, 'fraction_numerical': 0.007099247479767145, 'mean_word_length': 1.7102731819930743, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1768403', 'n_tokens_mistral': 2123, 'n_tokens_neox': 2042, 'n_words': 701}