<commit_before>require 'rubygems'
require 'ruby2ruby'
require 'set'

class RuleRewriter < Ruby2Ruby # :nodoc: all
  attr_accessor :rule_indx, :rules, :depends

  OP_LIST = Set.new([:<<, :<, :<=])
  TEMP_OP_LIST = Set.new([:-@, :~, :+@])
  MONOTONE_WHITELIST = Set.new([:==, :+, :<=, :-, :<, :>, :*, :~,
                                :pairs, :matches, :combos, :flatten,
                                :lefts, :rights, :map, :flat_map, :pro,
                                :cols, :key_cols, :val_cols, :payloads, :lambda,
                                :tabname, :ip_port, :port, :ip, :int_ip_port])

  def initialize(seed, bud_instance)
    @bud_instance = bud_instance
    @tables = {}
    @nm = false
    @rule_indx = seed
    @collect = false
    @rules = []
    @depends = []
    @nm_funcs_called = false
    super()
  end

  $not_id = [:not_coll_id]
  def resolve(obj, prefix, name)
    qn = prefix ? prefix + "." + name.to_s : name.to_s
    return [:collection, qn, obj.tables[name]] if obj.tables.has_key? name

    # does name refer to an import name?
    iobj = obj.import_instance name
    return [:import, qn, iobj] if iobj and iobj.respond_to? :tables

    return $not_id
  end

  def exp_id_type(recv, name, args) # call only if sexp type is :call
    return $not_id unless args.size == 1
    ty = $not_id
    if recv
      if recv.first == :call
        # possibly nested reference.
        rty, rqn, robj = exp_id_type(recv[1], recv[2], recv[3]) # rty, rqn, .. = receiver's type, qual name etc.
        ty = resolve(robj, rqn, name) if rty == :import
      end
    else
      # plain, un-prefixed name. See if it refers to a collection or import spec
      ty = resolve(@bud_instance, nil, name)
    end
    ty
  end

  def process_call(exp)
    recv, op, args = exp
    if OP_LIST.include?(op) and @context[1] == :block and @context.length == 4
      # NB: context.length is 4 when see a method call at the top-level of a
      # :defn block -- this is where we expect Bloom statements to appear
      do_rule(exp)
    else
      ty, qn, obj = exp_id_type(recv, op, args) # qn = qualified name, obj is the corresponding object
      if ty == :collection
        @tables[qn] = @nm if @collect
      #elsif ty == :import .. do nothing
      elsif ty == :not_coll_id
        # check if receiver is a collection, and further if the current exp
        # represents a field lookup
        op_is_field_name = false
        if recv and recv.first == :call
          rty, _, robj = exp_id_type(recv[1], recv[2], recv[3])
          if rty == :collection
            cols = robj.cols
            op_is_field_name = true if cols and cols.include?(op)
          end
        end
        # for CALM analysis, mark deletion rules as non-monotonic
        @nm = true if op == :-@
        if recv
          # don't worry about monotone ops, table names, table.attr calls, or
          # accessors of iterator variables
          unless RuleRewriter.is_monotone(op) or op_is_field_name or
                 recv.first == :lvar or op.to_s.start_with?("__")
            @nm = true
          end
        else
          # function called (implicit receiver = Bud instance) in a user-defined
          # code block. Check if it is non-monotonic (like budtime, that
          # produces a new value every time it is called)
          @nm_funcs_called = true unless RuleRewriter.is_monotone(op)
        end
      end
      if TEMP_OP_LIST.include? op
        @temp_op = op.to_s.gsub("@", "")
      end
      super
    end
  end

  def self.is_monotone(op)
    MONOTONE_WHITELIST.has_key?(op)
  end

  def collect_rhs(exp)
    @collect = true
    # rewrite constant array expressions to lambdas
    if exp[0] and exp[0] == :arglist
      # the <= case
      if exp[1] and exp[1][0] == :array
        exp = s(exp[0], s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, exp[1]))<commit_msg>Fix silly typo in monotone analysis.<commit_after>require 'rubygems'
require 'ruby2ruby'
require 'set'

class RuleRewriter < Ruby2Ruby # :nodoc: all
  attr_accessor :rule_indx, :rules, :depends

  OP_LIST = Set.new([:<<, :<, :<=])
  TEMP_OP_LIST = Set.new([:-@, :~, :+@])
  MONOTONE_WHITELIST = Set.new([:==, :+, :<=, :-, :<, :>, :*, :~,
                                :pairs, :matches, :combos, :flatten,
                                :lefts, :rights, :map, :flat_map, :pro,
                                :cols, :key_cols, :val_cols, :payloads, :lambda,
                                :tabname, :ip_port, :port, :ip, :int_ip_port])

  def initialize(seed, bud_instance)
    @bud_instance = bud_instance
    @tables = {}
    @nm = false
    @rule_indx = seed
    @collect = false
    @rules = []
    @depends = []
    @nm_funcs_called = false
    super()
  end

  $not_id = [:not_coll_id]
  def resolve(obj, prefix, name)
    qn = prefix ? prefix + "." + name.to_s : name.to_s
    return [:collection, qn, obj.tables[name]] if obj.tables.has_key? name

    # does name refer to an import name?
    iobj = obj.import_instance name
    return [:import, qn, iobj] if iobj and iobj.respond_to? :tables

    return $not_id
  end

  def exp_id_type(recv, name, args) # call only if sexp type is :call
    return $not_id unless args.size == 1
    ty = $not_id
    if recv
      if recv.first == :call
        # possibly nested reference.
        rty, rqn, robj = exp_id_type(recv[1], recv[2], recv[3]) # rty, rqn, .. = receiver's type, qual name etc.
        ty = resolve(robj, rqn, name) if rty == :import
      end
    else
      # plain, un-prefixed name. See if it refers to a collection or import spec
      ty = resolve(@bud_instance, nil, name)
    end
    ty
  end

  def process_call(exp)
    recv, op, args = exp
    if OP_LIST.include?(op) and @context[1] == :block and @context.length == 4
      # NB: context.length is 4 when see a method call at the top-level of a
      # :defn block -- this is where we expect Bloom statements to appear
      do_rule(exp)
    else
      ty, qn, _ = exp_id_type(recv, op, args) # qn = qualified name
      if ty == :collection
        @tables[qn] = @nm if @collect
      #elsif ty == :import .. do nothing
      elsif ty == :not_coll_id
        # check if receiver is a collection, and further if the current exp
        # represents a field lookup
        op_is_field_name = false
        if recv and recv.first == :call
          rty, _, robj = exp_id_type(recv[1], recv[2], recv[3])
          if rty == :collection
            cols = robj.cols
            op_is_field_name = true if cols and cols.include?(op)
          end
        end
        # for CALM analysis, mark deletion rules as non-monotonic
        @nm = true if op == :-@
        if recv
          # don't worry about monotone ops, table names, table.attr calls, or
          # accessors of iterator variables
          unless RuleRewriter.is_monotone(op) or op_is_field_name or
                 recv.first == :lvar or op.to_s.start_with?("__")
            @nm = true
          end
        else
          # function called (implicit receiver = Bud instance) in a user-defined
          # code block. Check if it is non-monotonic (like budtime, that
          # produces a new value every time it is called)
          @nm_funcs_called = true unless RuleRewriter.is_monotone(op)
        end
      end
      if TEMP_OP_LIST.include? op
        @temp_op = op.to_s.gsub("@", "")
      end
      super
    end
  end

  def self.is_monotone(op)
    MONOTONE_WHITELIST.include?(op)
  end

  def collect_rhs(exp)
    @collect = true
    # rewrite constant array expressions to lambdas
    if exp[0] and exp[0] == :arglist
      # the <= case
      if exp[1] and exp[1][0] == :array
        exp = s(exp[0], s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, exp[1]))
{'fraction_non_alphanumeric': 0.11716129032258064, 'fraction_numerical': 0.004903225806451613, 'mean_word_length': 2.184470008216927, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '721836', 'n_tokens_mistral': 2665, 'n_tokens_neox': 2578, 'n_words': 790}