<commit_before># encoding: utf-8
require 'rltk/ast'

class Class
  def basename
    name.match(/::(.*)$/).captures.first.downcase
  end
end

module Logical
  class Expression < RLTK::ASTNode; end

  class Variable < Expression
    value :name, String

    def inspect
      name
    end

    def evaluate(context)
      raise "Undefined variable '#{name}'" unless context.key?(name)
      context[name]
    end
  end

  class True < Expression
    def inspect
      "true"
    end

    def evaluate(context)
      true
    end
  end

  class False < Expression
    def inspect
      "false"
    end

    def evaluate(context)
      false
    end
  end

  class Define < Expression
    value :var,  String
    child :expr, Expression

    def inspect
      "#{var} := #{expr.inspect}"
    end

    def evaluate(context)
      context[var] = expr.evaluate(context)
    end
  end

  class UnaryFn < Expression
    child :arg, Expression

    def inspect
      self.class::SYMBOL + arg.inspect
    end
  end

  class BinaryFn < Expression
    child :left,  Expression
    child :right, Expression

    def inspect
      "(#{left.inspect} #{self.class::SYMBOL} #{right.inspect})"
    end
  end

  class Not < UnaryFn
    SYMBOL = "¬"
    def evaluate(context)
      not arg.evaluate(context)
    end<commit_msg>Add method to get the set of variables occuring in an AST.<commit_after># encoding: utf-8
require 'rltk/ast'

class Class
  def basename
    name.match(/::(.*)$/).captures.first.downcase
  end
end

module Logical
  class Expression < RLTK::ASTNode
    def vars
      []
    end
  end

  class Variable < Expression
    value :name, String

    def inspect
      name
    end

    def evaluate(context)
      raise "Undefined variable '#{name}'" unless context.key?(name)
      context[name]
    end

    def vars
      [name]
    end
  end

  class True < Expression
    def inspect
      "true"
    end

    def evaluate(context)
      true
    end
  end

  class False < Expression
    def inspect
      "false"
    end

    def evaluate(context)
      false
    end
  end

  class Define < Expression
    value :var,  String
    child :expr, Expression

    def inspect
      "#{var} := #{expr.inspect}"
    end

    def evaluate(context)
      context[var] = expr.evaluate(context)
    end
  end

  class UnaryFn < Expression
    child :arg, Expression

    def inspect
      self.class::SYMBOL + arg.inspect
    end

    def vars
      arg.vars
    end
  end

  class BinaryFn < Expression
    child :left,  Expression
    child :right, Expression

    def inspect
      "(#{left.inspect} #{self.class::SYMBOL} #{right.inspect})"
    end

    def vars
      (left.vars + right.vars).uniq
    end
  end

  class Not < UnaryFn
    SYMBOL = "¬"
    def evaluate(context)
      not arg.evaluate(context)
    end
{'fraction_non_alphanumeric': 0.08977998580553584, 'fraction_numerical': 0.0007097232079489, 'mean_word_length': 2.196145124716553, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4288899', 'n_tokens_mistral': 906, 'n_tokens_neox': 865, 'n_words': 268}