<commit_before>require 'pegrb'

module LetGrammar

  # Environment class. Basically just chained hash maps modeling the usual
  # lexical scope structure we all know and love. Not optimized at all for any
  # kind of special access patterns.
  class Env
    def initialize(inner, outer); @inner, @outer = inner, outer; end
    def [](val); @inner[val] || @outer[val]; end
    def []=(key, val); @inner[key] = val; end
    def increment; Env.new({}, self); end
  end

  # AST classes.
  class ConstExp < Struct.new(:value)
    def eval(_); self.value; end
  end

  # Common evaluation strategy so we abstract it.
  class ArithmeticOp < Struct.new(:expressions)
    # We again use lazy enumerators to not evaluate all the expressions. There might be
    # a type error along the way so evaluating all the expressions and then hitting a type
    # error is wasted effort. Evaluate only when necessary and die as soon as we have a type error.
    def eval(env, op)
      accumulator = self.expressions[0].eval(env)
      self.expressions.lazy.drop(1).each {|x| accumulator = accumulator.send(op, x.eval(env))}
      accumulator
    end
  end

  # Just need the right symbol to evaluate.
  class DiffExp < ArithmeticOp
    def eval(env); super(env, :-); end
  end
  class AddExp < ArithmeticOp
    def eval(env); super(env, :+); end
  end
  class MultExp < ArithmeticOp
    def eval(env); super(env, :*); end
  end
  class DivExp < ArithmeticOp
    def eval(env); super(env, :/); end
  end
  class ModExp < ArithmeticOp
    def eval(env); super(env, :%); end
  end

  # Similar kind of abstraction as for +ArithmeticOp+.
  class OrderOp < Struct.new(:expressions)
    # We use lazy enumerators for short circuiting the operations because we don't
    # need to evaluate all the expressions. We can bail as soon as we see a false result.
    # We are assuming a finite collection of expressions and in the absence of macros this
    # assumption holds.
    def eval(env, op)
      lazy_exprs = self.expressions.lazy
      pairs = lazy_exprs.zip(lazy_exprs.drop(1)).take(self.expressions.length - 1)
      pairs.each {|x, y| return false unless x.eval(env).send(op, y.eval(env))}
      true
    end
  end

  # Just need the right symbol to evaluate.
  class LessExp < OrderOp
    def eval(env); super(env, :<); end
  end
  class LessEqExp < OrderOp
    def eval(env); super(env, :<=); end
  end
  class EqExp < OrderOp
    def eval(env); super(env, :==); end
  end
  class GreaterExp < OrderOp
    def eval(env); super(env, :>); end
  end
  class GreaterEqExp < OrderOp
    def eval(env); super(env, :>=); end
  end

  class ZeroCheck < Struct.new(:expressions)
    def eval(env); self.expressions.all? {|x| x.eval(env) == 0}; end
  end

  class IfExp < Struct.new(:test, :true_branch, :false_branch)
    def eval(env)
      self.test.eval(env) ? self.true_branch.eval(env) : self.false_branch.eval(env)
    end
  end

  class Identifier < Struct.new(:value)<commit_msg>comments<commit_after>require 'pegrb'

module LetGrammar

  # Environment class. Basically just chained hash maps modeling the usual
  # lexical scope structure we all know and love.
  class Env
    def initialize(inner, outer); @inner, @outer = inner, outer; end
    def [](val); @inner[val] || @outer[val]; end
    def []=(key, val); @inner[key] = val; end
    def increment; Env.new({}, self); end
  end

  # AST classes.
  class ConstExp < Struct.new(:value)
    def eval(_); self.value; end
  end

  # Common evaluation for arithmetic operations so we abstract it.
  class ArithmeticOp < Struct.new(:expressions)
    # We use lazy enumerators to not evaluate all the expressions. There might be
    # a type error along the way so evaluating all the expressions and then hitting a type
    # error is wasted effort. Evaluate only when necessary and die as soon as we have a type error.
    def eval(env, op)
      accumulator = self.expressions[0].eval(env)
      self.expressions.lazy.drop(1).each {|x| accumulator = accumulator.send(op, x.eval(env))}
      accumulator
    end
  end

  # Delegate to +ArithmeticOp.eval+ with the proper operator symbol.
  class DiffExp < ArithmeticOp
    def eval(env); super(env, :-); end
  end
  class AddExp < ArithmeticOp
    def eval(env); super(env, :+); end
  end
  class MultExp < ArithmeticOp
    def eval(env); super(env, :*); end
  end
  class DivExp < ArithmeticOp
    def eval(env); super(env, :/); end
  end
  class ModExp < ArithmeticOp
    def eval(env); super(env, :%); end
  end

  # Similar kind of abstraction as for +ArithmeticOp+.
  class OrderOp < Struct.new(:expressions)
    # We use lazy enumerators for short circuiting the operations because we don't
    # need to evaluate all the expressions. We can bail as soon as we see a false result.
    def eval(env, op)
      lazy_exprs = self.expressions.lazy
      pairs = lazy_exprs.zip(lazy_exprs.drop(1)).take(self.expressions.length - 1)
      pairs.each {|x, y| return false unless x.eval(env).send(op, y.eval(env))}
      true
    end
  end

  # Delegate to +OrderOp.eval+ with the right operator symbol.
  class LessExp < OrderOp
    def eval(env); super(env, :<); end
  end
  class LessEqExp < OrderOp
    def eval(env); super(env, :<=); end
  end
  class EqExp < OrderOp
    def eval(env); super(env, :==); end
  end
  class GreaterExp < OrderOp
    def eval(env); super(env, :>); end
  end
  class GreaterEqExp < OrderOp
    def eval(env); super(env, :>=); end
  end

  class ZeroCheck < Struct.new(:expressions)
    def eval(env); self.expressions.all? {|x| x.eval(env) == 0}; end
  end

  class IfExp < Struct.new(:test, :true_branch, :false_branch)
    def eval(env)
      self.test.eval(env) ? self.true_branch.eval(env) : self.false_branch.eval(env)
    end
  end

  class Identifier < Struct.new(:value)
{'fraction_non_alphanumeric': 0.10992234685073339, 'fraction_numerical': 0.001725625539257981, 'mean_word_length': 3.277490774907749, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4596717', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1836, 'n_words': 646}