<commit_before>class MiqExpression
  require_nested :Tag
  include Vmdb::Logging
  attr_accessor :exp, :context_type, :preprocess_options

  BASE_TABLES = %w(
    AuditEvent
    AvailabilityZone
    BottleneckEvent
    ChargebackVm
    ChargebackContainerProject
    ChargebackContainerImage
    CloudResourceQuota
    CloudTenant
    CloudVolume
    Compliance
    ManageIQ::Providers::Foreman::ConfigurationManager::ConfiguredSystem
    ManageIQ::Providers::ConfigurationManager
    Container
    ContainerGroup
    ContainerImage
    ContainerImageRegistry
    ContainerNode
    ContainerProject
    ContainerReplicator
    ContainerRoute
    ContainerService
    ContainerTemplate
    ManageIQ::Providers::CloudManager
    EmsCluster
    EmsClusterPerformance
    EmsEvent
    ManageIQ::Providers::InfraManager
    ExtManagementSystem
    ExtManagementSystemPerformance
    Flavor
    Host
    HostAggregate
    HostPerformance
    MiqGroup
    MiqRegion
    MiqRequest
    MiqServer
    MiqTemplate
    MiqWorker
    OntapFileShare
    OntapLogicalDisk
    OntapStorageSystem
    OntapStorageVolume
    OntapVolumeMetricsRollup
    OrchestrationStack
    OrchestrationTemplate
    PolicyEvent
    ResourcePool
    SecurityGroup
    Service
    ServiceTemplate
    Storage
    StorageFile
    StoragePerformance
    Switch
    ManageIQ::Providers::CloudManager::Template
    ManageIQ::Providers::InfraManager::Template
    Tenant
    User
    VimPerformanceTrend
    Vm
    ManageIQ::Providers::CloudManager::Vm
    ManageIQ::Providers::InfraManager::Vm
    VmPerformance
    Zone
  )

  INCLUDE_TABLES = %w(
    advanced_settings
    audit_events
    availability_zones
    cloud_networks
    cloud_resource_quotas
    cloud_tenants
    compliances
    compliance_details
    computer_systems
    configuration_profiles
    configuration_managers
    configured_systems
    containers
    container_groups
    container_projects
    container_images
    container_nodes
    customization_scripts
    customization_script_media
    customization_script_ptables
    disks
    ems_events
    ems_clusters
    ems_custom_attributes
    evm_owners
    event_logs
    ext_management_systems
    filesystem_drivers
    filesystems
    firewall_rules
    flavors
    groups
    guest_applications
    guest_devices
    hardwares
    hosts
    host_aggregates
    host_services
    kernel_drivers
    key_pairs
    lans
    last_compliances
    linux_initprocesses
    miq_actions
    miq_approval_stamps
    miq_custom_attributes
    miq_policy_sets
    miq_provisions
    miq_regions
    miq_requests
    miq_scsi_luns
    miq_servers
    miq_workers
    ontap_concrete_extents
    ontap_file_shares
    ontap_logical_disks
    ontap_plex_extents
    ontap_raid_group_extents
    ontap_storage_systems
    ontap_storage_volumes
    openscap_results
    openscap_rule_results
    orchestration_stack_outputs
    orchestration_stack_parameters
    orchestration_stack_resources
    orchestration_templates
    operating_system_flavors
    partitions
    ports
    processes
    miq_provision_templates
    miq_provision_vms
    miq_templates
    networks
    nics
    operating_systems
    patches
    registry_items
    resource_pools
    security_groups
    service_templates
    services
    snapshots
    stacks
    storages
    storage_adapters
    storage_files
    switches
    tenant_quotas
    users
    vms
    volumes
    win32_services
    zones
    storage_systems
    file_systems
    hosted_file_shares
    file_shares
    logical_disks
    storage_volumes
    base_storage_extents
    top_storage_extents
  )

  EXCLUDE_COLUMNS = %w(
    ^.*_id$
    ^id$
    ^min_derived_storage.*$
    ^max_derived_storage.*$
    assoc_ids
    capture_interval
    filters
    icon

    intervals_in_rollup

    max_cpu_ready_delta_summation
    max_cpu_system_delta_summation
    max_cpu_used_delta_summation
    max_cpu_wait_delta_summation
    max_derived_cpu_available
    max_derived_cpu_reserved
    max_derived_memory_available
    max_derived_memory_reserved

    memory_usage

    min_cpu_ready_delta_summation
    min_cpu_system_delta_summation
    min_cpu_used_delta_summation
    min_cpu_wait_delta_summation
    min_derived_memory_available
    min_derived_memory_reserved
    min_derived_cpu_available
    min_derived_cpu_reserved

    min_max

    options
    password
    policy_settings
    ^reserved$
    resource_id
    settings
    tag_names
    v_qualified_desc
  )

  EXCLUDE_EXCEPTIONS = %w(
    capacity_profile_1_memory_per_vm_with_min_max
    capacity_profile_1_vcpu_per_vm_with_min_max
    capacity_profile_2_memory_per_vm_with_min_max
    capacity_profile_2_vcpu_per_vm_with_min_max
    chain_id
    guid
    openscap_id
  )

  TAG_CLASSES = {
    'ManageIQ::Providers::CloudManager'           => 'ext_management_system',
    'EmsCluster'                                  => 'ems_cluster',
    'ManageIQ::Providers::InfraManager'           => 'ext_management_system',
    'ManageIQ::Providers::ContainerManager'       => 'ext_management_system',
    'ExtManagementSystem'                         => 'ext_management_system',
    'Host'                                        => 'host',
    'MiqGroup'                                    => 'miq_group',
    'MiqTemplate'                                 => 'miq_template',
    'ResourcePool'                                => 'resource_pool',
    'Service'                                     => 'service',
    'Storage'                                     => 'storage',
    'ManageIQ::Providers::CloudManager::Template' => 'miq_template',
    'ManageIQ::Providers::InfraManager::Template' => 'miq_template',
    'User'                                        => 'user',
    'Vm'                                          => 'vm',
    'VmOrTemplate'                                => 'vm',
    'ManageIQ::Providers::CloudManager::Vm'       => 'vm',
    'ManageIQ::Providers::InfraManager::Vm'       => 'vm',
    'ContainerProject'                            => 'container_project',
    'ContainerImage'                              => 'container_image'
  }
  EXCLUDE_FROM_RELATS = {
    "ManageIQ::Providers::CloudManager" => ["hosts", "ems_clusters", "resource_pools"]
  }

  FORMAT_SUB_TYPES = {
    :boolean     => {
      :short_name => _("Boolean"),
      :title      => _("Enter true or false")
    },
    :bytes       => {
      :short_name => _("Bytes"),
      :title      => _("Enter the number of Bytes"),
      :units      => [
        [_("Bytes"), :bytes],
        [_("KB"), :kilobytes],
        [_("MB"), :megabytes],
        [_("GB"), :gigabytes],
        [_("TB"), :terabytes]
      ]
    },
    :date        => {
      :short_name => _("Date"),
      :title      => _("Click to Choose a Date")
    },
    :datetime    => {
      :short_name => _("Date / Time"),
      :title      => _("Click to Choose a Date / Time")
    },
    :float       => {
      :short_name => _("Number"),
      :title      => _("Enter a Number (like 12.56)")
    },
    :gigabytes   => {
      :short_name => _("Gigabytes"),
      :title      => _("Enter the number of Gigabytes")
    },
    :integer     => {
      :short_name => _("Integer"),
      :title      => _("Enter an Integer")
    },
    :kbps        => {
      :short_name => _("KBps"),
      :title      => _("Enter the Kilobytes per second")
    },
    :kilobytes   => {
      :short_name => _("Kilobytes"),
      :title      => _("Enter the number of Kilobytes")
    },
    :megabytes   => {
      :short_name => _("Megabytes"),
      :title      => _("Enter the number of Megabytes")
    },
    :mhz         => {
      :short_name => _("Mhz"),
      :title      => _("Enter the number of Megahertz")
    },
    :numeric_set => {
      :short_name => _("Number List"),
      :title      => _("Enter a list of numbers separated by commas")
    },
    :percent     => {
      :short_name => _("Percent"),
      :title      => _("Enter a Percent (like 12.5)"),
    },
    :regex       => {
      :short_name => _("Regular Expression"),
      :title      => _("Enter a Regular Expression")
    },
    :string      => {
      :short_name => _("Text String"),
      :title      => _("Enter a Text String")
    },
    :string_set  => {
      :short_name => _("String List"),
      :title      => _("Enter a list of text strings separated by commas")
    }
  }
  FORMAT_SUB_TYPES[:fixnum] = FORMAT_SUB_TYPES[:decimal] = FORMAT_SUB_TYPES[:integer]
  FORMAT_SUB_TYPES[:mhz_avg] = FORMAT_SUB_TYPES[:mhz]
  FORMAT_SUB_TYPES[:text] = FORMAT_SUB_TYPES[:string]
  FORMAT_BYTE_SUFFIXES = FORMAT_SUB_TYPES[:bytes][:units].inject({}) { |h, (v, k)| h[k] = v; h }
  BYTE_FORMAT_WHITELIST = Hash[FORMAT_BYTE_SUFFIXES.keys.collect(&:to_s).zip(FORMAT_BYTE_SUFFIXES.keys)]

  def initialize(exp, ctype = nil)
    @exp = exp
    @context_type = ctype

    load_virtual_custom_attributes
  end

  def load_virtual_custom_attributes
    return unless @exp

    custom_attributes_group_by_model = custom_attribute_columns_and_models.compact.group_by { |x| x[:model] }

    custom_attributes_group_by_model.each do |model, custom_attribute|
      model.load_custom_attributes_for(custom_attribute.map { |x| x[:column] })
    end
  end

  def self.proto?
    return @proto if defined?(@proto)
    @proto = ::Settings.product.proto
  end

  def self.to_human(exp)
    if exp.kind_of?(self)
      exp.to_human
    else
      if exp.kind_of?(Hash)
        case exp["mode"]
        when "tag_expr"
          return exp["expr"]
        when "tag"
          tag = [exp["ns"], exp["tag"]].join("/")
          if exp["include"] == "none"
            return "Not Tagged With #{tag}"
          else
            return "Tagged With #{tag}"
          end
        when "script"
          if exp["expr"] == "true"
            return "Always True"
          else
            return exp["expr"]
          end
        else
          return new(exp).to_human
        end
      else
        return exp.inspect
      end
    end
  end

  def to_human
    self.class._to_human(@exp)
  end

  def self._to_human(exp, options = {})
    return exp unless exp.kind_of?(Hash) || exp.kind_of?(Array)

    keys = exp.keys
    keys.delete(:token)
    operator = keys.first
    case operator.downcase
    when "like", "not like", "starts with", "ends with", "includes", "includes any", "includes all", "includes only", "limited to", "regular expression", "regular expression matches", "regular expression does not match", "equal", "=", "<", ">", ">=", "<=", "!=", "before", "after"
      operands = operands2humanvalue(exp[operator], options)
      clause = operands.join(" #{normalize_operator(operator)} ")
    when "and", "or"
      clause = "( " + exp[operator].collect { |operand| _to_human(operand) }.join(" #{normalize_operator(operator)} ") + " )"
    when "not", "!"
      clause = normalize_operator(operator) + " ( " + _to_human(exp[operator]) + " )"
    when "is null", "is not null", "is empty", "is not empty"
      clause = operands2humanvalue(exp[operator], options).first + " " + operator
    when "contains"
      operands = operands2humanvalue(exp[operator], options)
      clause = operands.join(" #{normalize_operator(operator)} ")
    when "find"
      # FIND Vm.users-name = 'Administrator' CHECKALL Vm.users-enabled = 1
      check = nil
      check = "checkall" if exp[operator].include?("checkall")
      check = "checkany" if exp[operator].include?("checkany")
      check = "checkcount" if exp[operator].include?("checkcount")
      raise _("expression malformed,  must contain one of 'checkall', 'checkany', 'checkcount'") unless check
      check =~ /^check(.*)$/; mode = $1.upcase
      clause = "FIND" + " " + _to_human(exp[operator]["search"]) + " CHECK " + mode + " " + _to_human(exp[operator][check], :include_table => false).strip
    when "key exists"
      clause = "KEY EXISTS #{exp[operator]['regkey']}"
    when "value exists"
      clause = "VALUE EXISTS #{exp[operator]['regkey']} : #{exp[operator]['regval']}"
    when "is"
      operands = operands2humanvalue(exp[operator], options)
      clause = "#{operands.first} #{operator} #{operands.last}"
    when "between dates", "between times"
      col_name = exp[operator]["field"]
      col_type = get_col_type(col_name)
      col_human, dumy = operands2humanvalue(exp[operator], options)
      vals_human = exp[operator]["value"].collect { |v| quote_human(v, col_type) }
      clause = "#{col_human} #{operator} #{vals_human.first} AND #{vals_human.last}"
    when "from"
      col_name = exp[operator]["field"]
      col_type = get_col_type(col_name)
      col_human, dumy = operands2humanvalue(exp[operator], options)
      vals_human = exp[operator]["value"].collect { |v| quote_human(v, col_type) }
      clause = "#{col_human} #{operator} #{vals_human.first} THROUGH #{vals_human.last}"
    end

    # puts "clause: #{clause}"
    clause
  end

  def to_ruby(tz = nil)
    tz ||= "UTC"
    @ruby ||= self.class._to_ruby(@exp.deep_clone, @context_type, tz)
    @ruby.dup
  end

  def self._to_ruby(exp, context_type, tz)
    return exp unless exp.kind_of?(Hash)

    operator = exp.keys.first
    case operator.downcase
    when "equal", "=", "<", ">", ">=", "<=", "!="
      col_type = get_col_type(exp[operator]["field"]) if exp[operator]["field"]
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
    when "before"
      col_type = get_col_type(exp[operator]["field"]) if exp[operator]["field"]
      col_name = exp[operator]["field"]
      col_ruby, = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      val = exp[operator]["value"]
      clause = ruby_for_date_compare(col_ruby, col_type, tz, "<", val)
    when "after"
      col_type = get_col_type(exp[operator]["field"]) if exp[operator]["field"]
      col_name = exp[operator]["field"]
      col_ruby, = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      val = exp[operator]["value"]
      clause = ruby_for_date_compare(col_ruby, col_type, tz, nil, nil, ">", val)
    when "includes all"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = "(#{operands[0]} & #{operands[1]}) == #{operands[1]}"
    when "includes any"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = "(#{operands[1]} - #{operands[0]}) != #{operands[1]}"
    when "includes only", "limited to"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = "(#{operands[0]} - #{operands[1]}) == []"
    when "like", "not like", "starts with", "ends with", "includes"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      case operator.downcase
      when "starts with"
        operands[1] = "/^" + re_escape(operands[1].to_s) + "/"
      when "ends with"
        operands[1] = "/" + re_escape(operands[1].to_s) + "$/"
      else
        operands[1] = "/" + re_escape(operands[1].to_s) + "/"
      end
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
      clause = "!(" + clause + ")" if operator.downcase == "not like"
    when "regular expression matches", "regular expression does not match"
      operands = operands2rubyvalue(operator, exp[operator], context_type)

      # If it looks like a regular expression, sanitize from forward
      # slashes and interpolation
      #
      # Regular expressions with a single option are also supported,
      # e.g. "/abc/i"
      #
      # Otherwise sanitize the whole string and add the delimiters
      #
      # TODO: support regexes with more than one option
      if operands[1].starts_with?("/") && operands[1].ends_with?("/")
        operands[1][1..-2] = sanitize_regular_expression(operands[1][1..-2])
      elsif operands[1].starts_with?("/") && operands[1][-2] == "/"
        operands[1][1..-3] = sanitize_regular_expression(operands[1][1..-3])
      else
        operands[1] = "/" + sanitize_regular_expression(operands[1].to_s) + "/"
      end
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
    when "and", "or"
      clause = "(" + exp[operator].collect { |operand| _to_ruby(operand, context_type, tz) }.join(" #{normalize_ruby_operator(operator)} ") + ")"
    when "not", "!"
      clause = normalize_ruby_operator(operator) + "(" + _to_ruby(exp[operator], context_type, tz) + ")"
    when "is null", "is not null", "is empty", "is not empty"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
    when "contains"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = operands.join(" #{normalize_operator(operator)} ")
    when "find"
      # FIND Vm.users-name = 'Administrator' CHECKALL Vm.users-enabled = 1
      check = nil
      check = "checkall" if exp[operator].include?("checkall")
      check = "checkany" if exp[operator].include?("checkany")
      if exp[operator].include?("checkcount")
        check = "checkcount"
        op = exp[operator][check].keys.first
        exp[operator][check][op]["field"] = "<count>"
      end
      raise _("expression malformed,  must contain one of 'checkall', 'checkany', 'checkcount'") unless check
      check =~ /^check(.*)$/; mode = $1.downcase
      clause = "<find><search>" + _to_ruby(exp[operator]["search"], context_type, tz) + "</search><check mode=#{mode}>" + _to_ruby(exp[operator][check], context_type, tz) + "</check></find>"
    when "key exists"
      clause = operands2rubyvalue(operator, exp[operator], context_type)
    when "value exists"
      clause = operands2rubyvalue(operator, exp[operator], context_type)
    when "is"
      col_name = exp[operator]["field"]
      col_ruby, dummy = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      col_type = get_col_type(col_name)
      value = exp[operator]["value"]
      clause = if col_type == :date && !RelativeDatetime.relative?(value)
                 ruby_for_date_compare(col_ruby, col_type, tz, "==", value)
               else
                 ruby_for_date_compare(col_ruby, col_type, tz, ">=", value, "<=", value)
               end
    when "from"
      col_name = exp[operator]["field"]
      col_ruby, dummy = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      col_type = get_col_type(col_name)

      start_val, end_val = exp[operator]["value"]
      clause = ruby_for_date_compare(col_ruby, col_type, tz, ">=", start_val, "<=", end_val)
    else
      raise _("operator '%{operator_name}' is not supported") % {:operator_name => operator}
    end

    # puts "clause: #{clause}"
    clause
  end

  def to_sql(tz = nil)
    tz ||= "UTC"
    @pexp, attrs = preprocess_for_sql(@exp.deep_clone)
    sql = to_arel(@pexp, tz).to_sql if @pexp.present?
    incl = includes_for_sql unless sql.blank?
    [sql, incl, attrs]
  end

  def preprocess_for_sql(exp, attrs = nil)
    attrs ||= {:supported_by_sql => true}
    operator = exp.keys.first
    case operator.downcase
    when "and"
      exp[operator].dup.each { |atom| preprocess_for_sql(atom, attrs) }
      exp[operator].reject!(&:blank?)
      exp.delete(operator) if exp[operator].empty?
    when "or"
      or_attrs = {:supported_by_sql => true}
      exp[operator].each { |atom| preprocess_for_sql(atom, or_attrs) }
      exp[operator].reject!(&:blank?)
      attrs.merge!(or_attrs)
      exp.delete(operator) if !or_attrs[:supported_by_sql] || exp[operator].empty? # Clean out unsupported or empty operands
    when "not", "!"
      preprocess_for_sql(exp[operator], attrs)
      exp.delete(operator) if exp[operator].empty? # Clean out empty operands
    else
      # check operands to see if they can be represented in sql
      unless sql_supports_atom?(exp)
        attrs[:supported_by_sql] = false
        exp.delete(operator)
      end
    end

    exp.empty? ? [nil, attrs] : [exp, attrs]
  end

  def sql_supports_atom?(exp)
    operator = exp.keys.first
    case operator.downcase
    when "contains"
      if exp[operator].keys.include?("tag") && exp[operator]["tag"].split(".").length == 2 # Only support for tags of the main model
        return true
      elsif exp[operator].keys.include?("field") && exp[operator]["field"].split(".").length == 2
        db, field = exp[operator]["field"].split(".")
        assoc, field = field.split("-")
        ref = db.constantize.reflect_on_association(assoc.to_sym)
        return false unless ref
        return false unless ref.macro == :has_many || ref.macro == :has_one
        return false if ref.options && ref.options.key?(:as)
        return field_in_sql?(exp[operator]["field"])
      else
        return false
      end
    when "includes"
      # Support includes operator using "LIKE" only if first operand is in main table
      if exp[operator].key?("field") && (!exp[operator]["field"].include?(".") || (exp[operator]["field"].include?(".") && exp[operator]["field"].split(".").length == 2))
        return field_in_sql?(exp[operator]["field"])
      else
        # TODO: Support includes operator for sub-sub-tables
        return false
      end
    when "find", "regular expression matches", "regular expression does not match", "key exists", "value exists"
      return false
    else
      # => false if operand is a tag
      return false if exp[operator].keys.include?("tag")

      # => false if operand is a registry
      return false if exp[operator].keys.include?("regkey")

      # => TODO: support count of child relationship
      return false if exp[operator].key?("count")

      return field_in_sql?(exp[operator]["field"]) && value_in_sql?(exp[operator]["value"])
    end
  end

  def value_in_sql?(value)
    !Field.is_field?(value) || Field.parse(value).attribute_supported_by_sql?
  end

  def field_in_sql?(field)
    # => false if operand is from a virtual reflection
    return false if self.field_from_virtual_reflection?(field)
    return false unless attribute_supported_by_sql?(field)

    # => false if excluded by special case defined in preprocess options
    return false if self.field_excluded_by_preprocess_options?(field)

    true
  end

  def field_from_virtual_reflection?(field)
    col_details[field][:virtual_reflection]
  end

  def attribute_supported_by_sql?(field)
    return false unless col_details[field]
    col_details[field][:sql_support]
  end

  def field_excluded_by_preprocess_options?(field)
    col_details[field][:excluded_by_preprocess_options]
  end

  def col_details
    @col_details ||= self.class.get_cols_from_expression(@exp, @preprocess_options)
  end

  def includes_for_sql
    col_details.values.each_with_object({}) { |v, result| result.deep_merge!(v[:include]) }
  end

  def self.expand_conditional_clause(klass, cond)
    return klass.send(:sanitize_sql_for_conditions, cond) unless cond.kind_of?(Hash)

    cond = klass.predicate_builder.resolve_column_aliases(cond)
    cond = klass.send(:expand_hash_conditions_for_aggregates, cond)

    klass.predicate_builder.build_from_hash(cond).map { |b|
      klass.connection.visitor.compile b
    }.join(' AND ')
  end

  def self.merge_where_clauses(*list)
    list = list.compact.collect do |s|
      expand_conditional_clause(MiqReport, s)
    end.compact

    if list.size == 0
      nil
    elsif list.size == 1
      list.first
    else
      "(#{list.join(") AND (")})"
    end
  end

  def self.merge_includes(*incl_list)
    return nil if incl_list.blank?
    incl_list.compact.each_with_object({}) { |i, result| result.deep_merge!(i) }
  end

  def self.get_cols_from_expression(exp, options = {})
    result = {}
    if exp.kind_of?(Hash)
      if exp.key?("field")
        result[exp["field"]] = get_col_info(exp["field"], options) unless exp["field"] == "<count>"
      elsif exp.key?("count")
        result[exp["count"]] = get_col_info(exp["count"], options)
      elsif exp.key?("tag")
        # ignore
      else
        exp.each_value { |v| result.merge!(get_cols_from_expression(v, options)) }
      end
    elsif exp.kind_of?(Array)
      exp.each { |v| result.merge!(get_cols_from_expression(v, options)) }
    end
    result
  end

  def self.get_col_info(field, options = {})
    result ||= {:data_type => nil, :virtual_reflection => false, :virtual_column => false, :sql_support => true, :excluded_by_preprocess_options => false, :tag => false, :include => {}}
    col = field.split("-").last if field.include?("-")
    parts = field.split("-").first.split(".")
    model = parts.shift

    if model.downcase == "managed" || parts.last == "managed"
      result[:data_type] = :string
      result[:tag] = true
      return result
    end
    model = model_class(model)
    cur_incl = result[:include]

    parts.each do |assoc|
      assoc = assoc.to_sym
      ref = model.reflection_with_virtual(assoc)
      result[:virtual_reflection] = true if model.virtual_reflection?(assoc)

      unless result[:virtual_reflection]
        cur_incl[assoc] ||= {}
        cur_incl = cur_incl[assoc]
      end

      unless ref
        result[:virtual_reflection] = true
        result[:sql_support] = false
        result[:virtual_column] = true
        return result
      end

      model = ref.klass
    end
    if col
      result[:data_type] = col_type(model, col)
      result[:format_sub_type] = MiqReport::Formats.sub_type(col.to_sym) || result[:data_type]
      result[:virtual_column] = model.virtual_attribute?(col.to_s)
      result[:sql_support] = model.attribute_supported_by_sql?(col.to_s)
      result[:excluded_by_preprocess_options] = self.exclude_col_by_preprocess_options?(col, options)
    end
    result
  end

  def self.exclude_col_by_preprocess_options?(col, options)
    return false unless options.kind_of?(Hash)
    return false unless options[:vim_performance_daily_adhoc]
    Metric::Rollup.excluded_col_for_expression?(col.to_sym)
  end

  def lenient_evaluate(obj, tz = nil)
    ruby_exp = to_ruby(tz)
    ruby_exp.nil? || Condition.subst_matches?(ruby_exp, obj)
  end

  def evaluate(obj, tz = nil)
    ruby_exp = to_ruby(tz)
    _log.debug("Expression before substitution: #{ruby_exp}")
    subst_expr = Condition.subst(ruby_exp, obj)
    _log.debug("Expression after substitution: #{subst_expr}")
    result = Condition.do_eval(subst_expr)
    _log.debug("Expression evaluation result: [#{result}]")
    result
  end

  def self.evaluate_atoms(exp, obj)
    exp = exp.kind_of?(self) ? copy_hash(exp.exp) : exp
    exp["result"] = new(exp).evaluate(obj)

    operators = exp.keys
    operators.each do|k|
      if ["and", "or"].include?(k.to_s.downcase)      # and/or atom is an array of atoms
        exp[k].each do|atom|
          evaluate_atoms(atom, obj)
        end
      elsif ["not", "!"].include?(k.to_s.downcase)    # not atom is a hash expression
        evaluate_atoms(exp[k], obj)
      else
        next
      end
    end
    exp
  end

  def self.operands2humanvalue(ops, options = {})
    # puts "Enter: operands2humanvalue: ops: #{ops.inspect}"
    ret = []
    if ops["tag"]
      v = nil
      ret.push(ops["alias"] || value2human(ops["tag"], options))
      MiqExpression.get_entry_details(ops["tag"]).each do|t|
        v = "'" + t.first + "'" if t.last == ops["value"]
      end
      if ops["value"] == :user_input
        v = "<user input>"
      else
        v ||= ops["value"].kind_of?(String) ? "'" + ops["value"] + "'" : ops["value"]
      end
      ret.push(v)
    elsif ops["field"]
      ops["value"] ||= ''
      if ops["field"] == "<count>"
        ret.push(nil)
        ret.push(ops["value"])
      else
        ret.push(ops["alias"] || value2human(ops["field"], options))
        if ops["value"] == :user_input
          ret.push("<user input>")
        else
          col_type = get_col_type(ops["field"]) || "string"
          ret.push(quote_human(ops["value"], col_type.to_s))
        end
      end
    elsif ops["count"]
      ret.push("COUNT OF " + (ops["alias"] || value2human(ops["count"], options)).strip)
      if ops["value"] == :user_input
        ret.push("<user input>")
      else
        ret.push(ops["value"])
      end
    elsif ops["regkey"]
      ops["value"] ||= ''
      ret.push(ops["regkey"] + " : " + ops["regval"])
      ret.push(ops["value"].kind_of?(String) ? "'" + ops["value"] + "'" : ops["value"])
    elsif ops["value"]
      ret.push(nil)
      ret.push(ops["value"])
    end
    ret
  end

  def self.value2human(val, options = {})
    options = {
      :include_model => true,
      :include_table => true
    }.merge(options)
    tables, col = val.split("-")
    first = true
    val_is_a_tag = false
    ret = ""
    if options[:include_table] == true
      friendly = tables.split(".").collect do|t|
        if t.downcase == "managed"
          val_is_a_tag = true
          "#{Tenant.root_tenant.name} Tags"
        elsif t.downcase == "user_tag"
          "My Tags"
        else
          if first
            first = nil
            next unless options[:include_model] == true
            Dictionary.gettext(t, :type => :model, :notfound => :titleize)
          else
            Dictionary.gettext(t, :type => :table, :notfound => :titleize)
          end
        end
      end.compact
      ret = friendly.join(".")
      ret << " : " unless ret.blank? || col.blank?
    end
    if val_is_a_tag
      if col
        classification = options[:classification] || Classification.find_by_name(col)
        ret << (classification ? classification.description : col)
      end
    else
      model = tables.blank? ? nil : tables.split(".").last.singularize.camelize
      dict_col = model.nil? ? col : [model, col].join(".")
      column_human = if col
                       if col.starts_with?(CustomAttributeMixin::CUSTOM_ATTRIBUTES_PREFIX)
                         col.gsub(CustomAttributeMixin::CUSTOM_ATTRIBUTES_PREFIX, "")
                       else
                         Dictionary.gettext(dict_col, :type => :column, :notfound => :titleize)
                       end
                     end
      ret << column_human if col
    end
    ret = " #{ret}" unless ret.include?(":")
    ret
  end

  def self.operands2rubyvalue(operator, ops, context_type)
    # puts "Enter: operands2rubyvalue: operator: #{operator}, ops: #{ops.inspect}"
    operator = operator.downcase
    ops["tag"] = ops["field"] if operator == "contains" && !ops["tag"] # process values in contains as tags

    if ops["tag"] && context_type != "hash"
      ref, val = value2tag(preprocess_managed_tag(ops["tag"]), ops["value"])
      [ref ? "<exist ref=#{ref}>#{val}</exist>" : "<exist>#{val}</exist>"]
    elsif ops["tag"] && context_type == "hash"
      # This is only for supporting reporting "display filters"
      # In the report object the tag value is actually the description and not the raw tag name.
      # So we have to trick it by replacing the value with the description.
      description = MiqExpression.get_entry_details(ops["tag"]).inject("") do|s, t|
        break(t.first) if t.last == ops["value"]
        s
      end
      val = ops["tag"].split(".").last.split("-").join(".")
      fld = "<value type=string>#{val}</value>"
      [fld, quote(description, "string")]
    elsif ops["field"]
      if ops["field"] == "<count>"
        ["<count>", quote(ops["value"], "integer")]
      else
        col_type = get_col_type(ops["field"]) || "string"
        case context_type
        when "hash"
          val = ops["field"].split(".").last.split("-").join(".")
          fld = "<value type=#{col_type}>#{val}</value>"
        else
          ref, val = value2tag(ops["field"])
          fld = "<value ref=#{ref}, type=#{col_type}>#{val}</value>"
        end
        if ["like", "not like", "starts with", "ends with", "includes", "regular expression matches", "regular expression does not match"].include?(operator)
          [fld, ops["value"]]
        else
          [fld, quote(ops["value"], col_type.to_s)]
        end
      end
    elsif ops["count"]
      ref, count = value2tag(ops["count"])
      field = ref ? "<count ref=#{ref}>#{count}</count>" : "<count>#{count}</count>"
      [field, quote(ops["value"], "integer")]
    elsif ops["regkey"]
      if operator == "key exists"
        "<registry key_exists=1, type=boolean>#{ops["regkey"].strip}</registry>  == 'true'"
      elsif operator == "value exists"
        "<registry value_exists=1, type=boolean>#{ops["regkey"].strip} : #{ops["regval"]}</registry>  == 'true'"
      else
        fld = "<registry>#{ops["regkey"].strip} : #{ops["regval"]}</registry>"
        if ["like", "not like", "starts with", "ends with", "includes", "regular expression matches", "regular expression does not match"].include?(operator)
          [fld, ops["value"]]
        else
          [fld, quote(ops["value"], "string")]
        end
      end
    end
  end<commit_msg>`ref` cannot be `nil`, so eliminate branch<commit_after>class MiqExpression
  require_nested :Tag
  include Vmdb::Logging
  attr_accessor :exp, :context_type, :preprocess_options

  BASE_TABLES = %w(
    AuditEvent
    AvailabilityZone
    BottleneckEvent
    ChargebackVm
    ChargebackContainerProject
    ChargebackContainerImage
    CloudResourceQuota
    CloudTenant
    CloudVolume
    Compliance
    ManageIQ::Providers::Foreman::ConfigurationManager::ConfiguredSystem
    ManageIQ::Providers::ConfigurationManager
    Container
    ContainerGroup
    ContainerImage
    ContainerImageRegistry
    ContainerNode
    ContainerProject
    ContainerReplicator
    ContainerRoute
    ContainerService
    ContainerTemplate
    ManageIQ::Providers::CloudManager
    EmsCluster
    EmsClusterPerformance
    EmsEvent
    ManageIQ::Providers::InfraManager
    ExtManagementSystem
    ExtManagementSystemPerformance
    Flavor
    Host
    HostAggregate
    HostPerformance
    MiqGroup
    MiqRegion
    MiqRequest
    MiqServer
    MiqTemplate
    MiqWorker
    OntapFileShare
    OntapLogicalDisk
    OntapStorageSystem
    OntapStorageVolume
    OntapVolumeMetricsRollup
    OrchestrationStack
    OrchestrationTemplate
    PolicyEvent
    ResourcePool
    SecurityGroup
    Service
    ServiceTemplate
    Storage
    StorageFile
    StoragePerformance
    Switch
    ManageIQ::Providers::CloudManager::Template
    ManageIQ::Providers::InfraManager::Template
    Tenant
    User
    VimPerformanceTrend
    Vm
    ManageIQ::Providers::CloudManager::Vm
    ManageIQ::Providers::InfraManager::Vm
    VmPerformance
    Zone
  )

  INCLUDE_TABLES = %w(
    advanced_settings
    audit_events
    availability_zones
    cloud_networks
    cloud_resource_quotas
    cloud_tenants
    compliances
    compliance_details
    computer_systems
    configuration_profiles
    configuration_managers
    configured_systems
    containers
    container_groups
    container_projects
    container_images
    container_nodes
    customization_scripts
    customization_script_media
    customization_script_ptables
    disks
    ems_events
    ems_clusters
    ems_custom_attributes
    evm_owners
    event_logs
    ext_management_systems
    filesystem_drivers
    filesystems
    firewall_rules
    flavors
    groups
    guest_applications
    guest_devices
    hardwares
    hosts
    host_aggregates
    host_services
    kernel_drivers
    key_pairs
    lans
    last_compliances
    linux_initprocesses
    miq_actions
    miq_approval_stamps
    miq_custom_attributes
    miq_policy_sets
    miq_provisions
    miq_regions
    miq_requests
    miq_scsi_luns
    miq_servers
    miq_workers
    ontap_concrete_extents
    ontap_file_shares
    ontap_logical_disks
    ontap_plex_extents
    ontap_raid_group_extents
    ontap_storage_systems
    ontap_storage_volumes
    openscap_results
    openscap_rule_results
    orchestration_stack_outputs
    orchestration_stack_parameters
    orchestration_stack_resources
    orchestration_templates
    operating_system_flavors
    partitions
    ports
    processes
    miq_provision_templates
    miq_provision_vms
    miq_templates
    networks
    nics
    operating_systems
    patches
    registry_items
    resource_pools
    security_groups
    service_templates
    services
    snapshots
    stacks
    storages
    storage_adapters
    storage_files
    switches
    tenant_quotas
    users
    vms
    volumes
    win32_services
    zones
    storage_systems
    file_systems
    hosted_file_shares
    file_shares
    logical_disks
    storage_volumes
    base_storage_extents
    top_storage_extents
  )

  EXCLUDE_COLUMNS = %w(
    ^.*_id$
    ^id$
    ^min_derived_storage.*$
    ^max_derived_storage.*$
    assoc_ids
    capture_interval
    filters
    icon

    intervals_in_rollup

    max_cpu_ready_delta_summation
    max_cpu_system_delta_summation
    max_cpu_used_delta_summation
    max_cpu_wait_delta_summation
    max_derived_cpu_available
    max_derived_cpu_reserved
    max_derived_memory_available
    max_derived_memory_reserved

    memory_usage

    min_cpu_ready_delta_summation
    min_cpu_system_delta_summation
    min_cpu_used_delta_summation
    min_cpu_wait_delta_summation
    min_derived_memory_available
    min_derived_memory_reserved
    min_derived_cpu_available
    min_derived_cpu_reserved

    min_max

    options
    password
    policy_settings
    ^reserved$
    resource_id
    settings
    tag_names
    v_qualified_desc
  )

  EXCLUDE_EXCEPTIONS = %w(
    capacity_profile_1_memory_per_vm_with_min_max
    capacity_profile_1_vcpu_per_vm_with_min_max
    capacity_profile_2_memory_per_vm_with_min_max
    capacity_profile_2_vcpu_per_vm_with_min_max
    chain_id
    guid
    openscap_id
  )

  TAG_CLASSES = {
    'ManageIQ::Providers::CloudManager'           => 'ext_management_system',
    'EmsCluster'                                  => 'ems_cluster',
    'ManageIQ::Providers::InfraManager'           => 'ext_management_system',
    'ManageIQ::Providers::ContainerManager'       => 'ext_management_system',
    'ExtManagementSystem'                         => 'ext_management_system',
    'Host'                                        => 'host',
    'MiqGroup'                                    => 'miq_group',
    'MiqTemplate'                                 => 'miq_template',
    'ResourcePool'                                => 'resource_pool',
    'Service'                                     => 'service',
    'Storage'                                     => 'storage',
    'ManageIQ::Providers::CloudManager::Template' => 'miq_template',
    'ManageIQ::Providers::InfraManager::Template' => 'miq_template',
    'User'                                        => 'user',
    'Vm'                                          => 'vm',
    'VmOrTemplate'                                => 'vm',
    'ManageIQ::Providers::CloudManager::Vm'       => 'vm',
    'ManageIQ::Providers::InfraManager::Vm'       => 'vm',
    'ContainerProject'                            => 'container_project',
    'ContainerImage'                              => 'container_image'
  }
  EXCLUDE_FROM_RELATS = {
    "ManageIQ::Providers::CloudManager" => ["hosts", "ems_clusters", "resource_pools"]
  }

  FORMAT_SUB_TYPES = {
    :boolean     => {
      :short_name => _("Boolean"),
      :title      => _("Enter true or false")
    },
    :bytes       => {
      :short_name => _("Bytes"),
      :title      => _("Enter the number of Bytes"),
      :units      => [
        [_("Bytes"), :bytes],
        [_("KB"), :kilobytes],
        [_("MB"), :megabytes],
        [_("GB"), :gigabytes],
        [_("TB"), :terabytes]
      ]
    },
    :date        => {
      :short_name => _("Date"),
      :title      => _("Click to Choose a Date")
    },
    :datetime    => {
      :short_name => _("Date / Time"),
      :title      => _("Click to Choose a Date / Time")
    },
    :float       => {
      :short_name => _("Number"),
      :title      => _("Enter a Number (like 12.56)")
    },
    :gigabytes   => {
      :short_name => _("Gigabytes"),
      :title      => _("Enter the number of Gigabytes")
    },
    :integer     => {
      :short_name => _("Integer"),
      :title      => _("Enter an Integer")
    },
    :kbps        => {
      :short_name => _("KBps"),
      :title      => _("Enter the Kilobytes per second")
    },
    :kilobytes   => {
      :short_name => _("Kilobytes"),
      :title      => _("Enter the number of Kilobytes")
    },
    :megabytes   => {
      :short_name => _("Megabytes"),
      :title      => _("Enter the number of Megabytes")
    },
    :mhz         => {
      :short_name => _("Mhz"),
      :title      => _("Enter the number of Megahertz")
    },
    :numeric_set => {
      :short_name => _("Number List"),
      :title      => _("Enter a list of numbers separated by commas")
    },
    :percent     => {
      :short_name => _("Percent"),
      :title      => _("Enter a Percent (like 12.5)"),
    },
    :regex       => {
      :short_name => _("Regular Expression"),
      :title      => _("Enter a Regular Expression")
    },
    :string      => {
      :short_name => _("Text String"),
      :title      => _("Enter a Text String")
    },
    :string_set  => {
      :short_name => _("String List"),
      :title      => _("Enter a list of text strings separated by commas")
    }
  }
  FORMAT_SUB_TYPES[:fixnum] = FORMAT_SUB_TYPES[:decimal] = FORMAT_SUB_TYPES[:integer]
  FORMAT_SUB_TYPES[:mhz_avg] = FORMAT_SUB_TYPES[:mhz]
  FORMAT_SUB_TYPES[:text] = FORMAT_SUB_TYPES[:string]
  FORMAT_BYTE_SUFFIXES = FORMAT_SUB_TYPES[:bytes][:units].inject({}) { |h, (v, k)| h[k] = v; h }
  BYTE_FORMAT_WHITELIST = Hash[FORMAT_BYTE_SUFFIXES.keys.collect(&:to_s).zip(FORMAT_BYTE_SUFFIXES.keys)]

  def initialize(exp, ctype = nil)
    @exp = exp
    @context_type = ctype

    load_virtual_custom_attributes
  end

  def load_virtual_custom_attributes
    return unless @exp

    custom_attributes_group_by_model = custom_attribute_columns_and_models.compact.group_by { |x| x[:model] }

    custom_attributes_group_by_model.each do |model, custom_attribute|
      model.load_custom_attributes_for(custom_attribute.map { |x| x[:column] })
    end
  end

  def self.proto?
    return @proto if defined?(@proto)
    @proto = ::Settings.product.proto
  end

  def self.to_human(exp)
    if exp.kind_of?(self)
      exp.to_human
    else
      if exp.kind_of?(Hash)
        case exp["mode"]
        when "tag_expr"
          return exp["expr"]
        when "tag"
          tag = [exp["ns"], exp["tag"]].join("/")
          if exp["include"] == "none"
            return "Not Tagged With #{tag}"
          else
            return "Tagged With #{tag}"
          end
        when "script"
          if exp["expr"] == "true"
            return "Always True"
          else
            return exp["expr"]
          end
        else
          return new(exp).to_human
        end
      else
        return exp.inspect
      end
    end
  end

  def to_human
    self.class._to_human(@exp)
  end

  def self._to_human(exp, options = {})
    return exp unless exp.kind_of?(Hash) || exp.kind_of?(Array)

    keys = exp.keys
    keys.delete(:token)
    operator = keys.first
    case operator.downcase
    when "like", "not like", "starts with", "ends with", "includes", "includes any", "includes all", "includes only", "limited to", "regular expression", "regular expression matches", "regular expression does not match", "equal", "=", "<", ">", ">=", "<=", "!=", "before", "after"
      operands = operands2humanvalue(exp[operator], options)
      clause = operands.join(" #{normalize_operator(operator)} ")
    when "and", "or"
      clause = "( " + exp[operator].collect { |operand| _to_human(operand) }.join(" #{normalize_operator(operator)} ") + " )"
    when "not", "!"
      clause = normalize_operator(operator) + " ( " + _to_human(exp[operator]) + " )"
    when "is null", "is not null", "is empty", "is not empty"
      clause = operands2humanvalue(exp[operator], options).first + " " + operator
    when "contains"
      operands = operands2humanvalue(exp[operator], options)
      clause = operands.join(" #{normalize_operator(operator)} ")
    when "find"
      # FIND Vm.users-name = 'Administrator' CHECKALL Vm.users-enabled = 1
      check = nil
      check = "checkall" if exp[operator].include?("checkall")
      check = "checkany" if exp[operator].include?("checkany")
      check = "checkcount" if exp[operator].include?("checkcount")
      raise _("expression malformed,  must contain one of 'checkall', 'checkany', 'checkcount'") unless check
      check =~ /^check(.*)$/; mode = $1.upcase
      clause = "FIND" + " " + _to_human(exp[operator]["search"]) + " CHECK " + mode + " " + _to_human(exp[operator][check], :include_table => false).strip
    when "key exists"
      clause = "KEY EXISTS #{exp[operator]['regkey']}"
    when "value exists"
      clause = "VALUE EXISTS #{exp[operator]['regkey']} : #{exp[operator]['regval']}"
    when "is"
      operands = operands2humanvalue(exp[operator], options)
      clause = "#{operands.first} #{operator} #{operands.last}"
    when "between dates", "between times"
      col_name = exp[operator]["field"]
      col_type = get_col_type(col_name)
      col_human, dumy = operands2humanvalue(exp[operator], options)
      vals_human = exp[operator]["value"].collect { |v| quote_human(v, col_type) }
      clause = "#{col_human} #{operator} #{vals_human.first} AND #{vals_human.last}"
    when "from"
      col_name = exp[operator]["field"]
      col_type = get_col_type(col_name)
      col_human, dumy = operands2humanvalue(exp[operator], options)
      vals_human = exp[operator]["value"].collect { |v| quote_human(v, col_type) }
      clause = "#{col_human} #{operator} #{vals_human.first} THROUGH #{vals_human.last}"
    end

    # puts "clause: #{clause}"
    clause
  end

  def to_ruby(tz = nil)
    tz ||= "UTC"
    @ruby ||= self.class._to_ruby(@exp.deep_clone, @context_type, tz)
    @ruby.dup
  end

  def self._to_ruby(exp, context_type, tz)
    return exp unless exp.kind_of?(Hash)

    operator = exp.keys.first
    case operator.downcase
    when "equal", "=", "<", ">", ">=", "<=", "!="
      col_type = get_col_type(exp[operator]["field"]) if exp[operator]["field"]
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
    when "before"
      col_type = get_col_type(exp[operator]["field"]) if exp[operator]["field"]
      col_name = exp[operator]["field"]
      col_ruby, = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      val = exp[operator]["value"]
      clause = ruby_for_date_compare(col_ruby, col_type, tz, "<", val)
    when "after"
      col_type = get_col_type(exp[operator]["field"]) if exp[operator]["field"]
      col_name = exp[operator]["field"]
      col_ruby, = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      val = exp[operator]["value"]
      clause = ruby_for_date_compare(col_ruby, col_type, tz, nil, nil, ">", val)
    when "includes all"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = "(#{operands[0]} & #{operands[1]}) == #{operands[1]}"
    when "includes any"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = "(#{operands[1]} - #{operands[0]}) != #{operands[1]}"
    when "includes only", "limited to"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = "(#{operands[0]} - #{operands[1]}) == []"
    when "like", "not like", "starts with", "ends with", "includes"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      case operator.downcase
      when "starts with"
        operands[1] = "/^" + re_escape(operands[1].to_s) + "/"
      when "ends with"
        operands[1] = "/" + re_escape(operands[1].to_s) + "$/"
      else
        operands[1] = "/" + re_escape(operands[1].to_s) + "/"
      end
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
      clause = "!(" + clause + ")" if operator.downcase == "not like"
    when "regular expression matches", "regular expression does not match"
      operands = operands2rubyvalue(operator, exp[operator], context_type)

      # If it looks like a regular expression, sanitize from forward
      # slashes and interpolation
      #
      # Regular expressions with a single option are also supported,
      # e.g. "/abc/i"
      #
      # Otherwise sanitize the whole string and add the delimiters
      #
      # TODO: support regexes with more than one option
      if operands[1].starts_with?("/") && operands[1].ends_with?("/")
        operands[1][1..-2] = sanitize_regular_expression(operands[1][1..-2])
      elsif operands[1].starts_with?("/") && operands[1][-2] == "/"
        operands[1][1..-3] = sanitize_regular_expression(operands[1][1..-3])
      else
        operands[1] = "/" + sanitize_regular_expression(operands[1].to_s) + "/"
      end
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
    when "and", "or"
      clause = "(" + exp[operator].collect { |operand| _to_ruby(operand, context_type, tz) }.join(" #{normalize_ruby_operator(operator)} ") + ")"
    when "not", "!"
      clause = normalize_ruby_operator(operator) + "(" + _to_ruby(exp[operator], context_type, tz) + ")"
    when "is null", "is not null", "is empty", "is not empty"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = operands.join(" #{normalize_ruby_operator(operator)} ")
    when "contains"
      operands = operands2rubyvalue(operator, exp[operator], context_type)
      clause = operands.join(" #{normalize_operator(operator)} ")
    when "find"
      # FIND Vm.users-name = 'Administrator' CHECKALL Vm.users-enabled = 1
      check = nil
      check = "checkall" if exp[operator].include?("checkall")
      check = "checkany" if exp[operator].include?("checkany")
      if exp[operator].include?("checkcount")
        check = "checkcount"
        op = exp[operator][check].keys.first
        exp[operator][check][op]["field"] = "<count>"
      end
      raise _("expression malformed,  must contain one of 'checkall', 'checkany', 'checkcount'") unless check
      check =~ /^check(.*)$/; mode = $1.downcase
      clause = "<find><search>" + _to_ruby(exp[operator]["search"], context_type, tz) + "</search><check mode=#{mode}>" + _to_ruby(exp[operator][check], context_type, tz) + "</check></find>"
    when "key exists"
      clause = operands2rubyvalue(operator, exp[operator], context_type)
    when "value exists"
      clause = operands2rubyvalue(operator, exp[operator], context_type)
    when "is"
      col_name = exp[operator]["field"]
      col_ruby, dummy = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      col_type = get_col_type(col_name)
      value = exp[operator]["value"]
      clause = if col_type == :date && !RelativeDatetime.relative?(value)
                 ruby_for_date_compare(col_ruby, col_type, tz, "==", value)
               else
                 ruby_for_date_compare(col_ruby, col_type, tz, ">=", value, "<=", value)
               end
    when "from"
      col_name = exp[operator]["field"]
      col_ruby, dummy = operands2rubyvalue(operator, {"field" => col_name}, context_type)
      col_type = get_col_type(col_name)

      start_val, end_val = exp[operator]["value"]
      clause = ruby_for_date_compare(col_ruby, col_type, tz, ">=", start_val, "<=", end_val)
    else
      raise _("operator '%{operator_name}' is not supported") % {:operator_name => operator}
    end

    # puts "clause: #{clause}"
    clause
  end

  def to_sql(tz = nil)
    tz ||= "UTC"
    @pexp, attrs = preprocess_for_sql(@exp.deep_clone)
    sql = to_arel(@pexp, tz).to_sql if @pexp.present?
    incl = includes_for_sql unless sql.blank?
    [sql, incl, attrs]
  end

  def preprocess_for_sql(exp, attrs = nil)
    attrs ||= {:supported_by_sql => true}
    operator = exp.keys.first
    case operator.downcase
    when "and"
      exp[operator].dup.each { |atom| preprocess_for_sql(atom, attrs) }
      exp[operator].reject!(&:blank?)
      exp.delete(operator) if exp[operator].empty?
    when "or"
      or_attrs = {:supported_by_sql => true}
      exp[operator].each { |atom| preprocess_for_sql(atom, or_attrs) }
      exp[operator].reject!(&:blank?)
      attrs.merge!(or_attrs)
      exp.delete(operator) if !or_attrs[:supported_by_sql] || exp[operator].empty? # Clean out unsupported or empty operands
    when "not", "!"
      preprocess_for_sql(exp[operator], attrs)
      exp.delete(operator) if exp[operator].empty? # Clean out empty operands
    else
      # check operands to see if they can be represented in sql
      unless sql_supports_atom?(exp)
        attrs[:supported_by_sql] = false
        exp.delete(operator)
      end
    end

    exp.empty? ? [nil, attrs] : [exp, attrs]
  end

  def sql_supports_atom?(exp)
    operator = exp.keys.first
    case operator.downcase
    when "contains"
      if exp[operator].keys.include?("tag") && exp[operator]["tag"].split(".").length == 2 # Only support for tags of the main model
        return true
      elsif exp[operator].keys.include?("field") && exp[operator]["field"].split(".").length == 2
        db, field = exp[operator]["field"].split(".")
        assoc, field = field.split("-")
        ref = db.constantize.reflect_on_association(assoc.to_sym)
        return false unless ref
        return false unless ref.macro == :has_many || ref.macro == :has_one
        return false if ref.options && ref.options.key?(:as)
        return field_in_sql?(exp[operator]["field"])
      else
        return false
      end
    when "includes"
      # Support includes operator using "LIKE" only if first operand is in main table
      if exp[operator].key?("field") && (!exp[operator]["field"].include?(".") || (exp[operator]["field"].include?(".") && exp[operator]["field"].split(".").length == 2))
        return field_in_sql?(exp[operator]["field"])
      else
        # TODO: Support includes operator for sub-sub-tables
        return false
      end
    when "find", "regular expression matches", "regular expression does not match", "key exists", "value exists"
      return false
    else
      # => false if operand is a tag
      return false if exp[operator].keys.include?("tag")

      # => false if operand is a registry
      return false if exp[operator].keys.include?("regkey")

      # => TODO: support count of child relationship
      return false if exp[operator].key?("count")

      return field_in_sql?(exp[operator]["field"]) && value_in_sql?(exp[operator]["value"])
    end
  end

  def value_in_sql?(value)
    !Field.is_field?(value) || Field.parse(value).attribute_supported_by_sql?
  end

  def field_in_sql?(field)
    # => false if operand is from a virtual reflection
    return false if self.field_from_virtual_reflection?(field)
    return false unless attribute_supported_by_sql?(field)

    # => false if excluded by special case defined in preprocess options
    return false if self.field_excluded_by_preprocess_options?(field)

    true
  end

  def field_from_virtual_reflection?(field)
    col_details[field][:virtual_reflection]
  end

  def attribute_supported_by_sql?(field)
    return false unless col_details[field]
    col_details[field][:sql_support]
  end

  def field_excluded_by_preprocess_options?(field)
    col_details[field][:excluded_by_preprocess_options]
  end

  def col_details
    @col_details ||= self.class.get_cols_from_expression(@exp, @preprocess_options)
  end

  def includes_for_sql
    col_details.values.each_with_object({}) { |v, result| result.deep_merge!(v[:include]) }
  end

  def self.expand_conditional_clause(klass, cond)
    return klass.send(:sanitize_sql_for_conditions, cond) unless cond.kind_of?(Hash)

    cond = klass.predicate_builder.resolve_column_aliases(cond)
    cond = klass.send(:expand_hash_conditions_for_aggregates, cond)

    klass.predicate_builder.build_from_hash(cond).map { |b|
      klass.connection.visitor.compile b
    }.join(' AND ')
  end

  def self.merge_where_clauses(*list)
    list = list.compact.collect do |s|
      expand_conditional_clause(MiqReport, s)
    end.compact

    if list.size == 0
      nil
    elsif list.size == 1
      list.first
    else
      "(#{list.join(") AND (")})"
    end
  end

  def self.merge_includes(*incl_list)
    return nil if incl_list.blank?
    incl_list.compact.each_with_object({}) { |i, result| result.deep_merge!(i) }
  end

  def self.get_cols_from_expression(exp, options = {})
    result = {}
    if exp.kind_of?(Hash)
      if exp.key?("field")
        result[exp["field"]] = get_col_info(exp["field"], options) unless exp["field"] == "<count>"
      elsif exp.key?("count")
        result[exp["count"]] = get_col_info(exp["count"], options)
      elsif exp.key?("tag")
        # ignore
      else
        exp.each_value { |v| result.merge!(get_cols_from_expression(v, options)) }
      end
    elsif exp.kind_of?(Array)
      exp.each { |v| result.merge!(get_cols_from_expression(v, options)) }
    end
    result
  end

  def self.get_col_info(field, options = {})
    result ||= {:data_type => nil, :virtual_reflection => false, :virtual_column => false, :sql_support => true, :excluded_by_preprocess_options => false, :tag => false, :include => {}}
    col = field.split("-").last if field.include?("-")
    parts = field.split("-").first.split(".")
    model = parts.shift

    if model.downcase == "managed" || parts.last == "managed"
      result[:data_type] = :string
      result[:tag] = true
      return result
    end
    model = model_class(model)
    cur_incl = result[:include]

    parts.each do |assoc|
      assoc = assoc.to_sym
      ref = model.reflection_with_virtual(assoc)
      result[:virtual_reflection] = true if model.virtual_reflection?(assoc)

      unless result[:virtual_reflection]
        cur_incl[assoc] ||= {}
        cur_incl = cur_incl[assoc]
      end

      unless ref
        result[:virtual_reflection] = true
        result[:sql_support] = false
        result[:virtual_column] = true
        return result
      end

      model = ref.klass
    end
    if col
      result[:data_type] = col_type(model, col)
      result[:format_sub_type] = MiqReport::Formats.sub_type(col.to_sym) || result[:data_type]
      result[:virtual_column] = model.virtual_attribute?(col.to_s)
      result[:sql_support] = model.attribute_supported_by_sql?(col.to_s)
      result[:excluded_by_preprocess_options] = self.exclude_col_by_preprocess_options?(col, options)
    end
    result
  end

  def self.exclude_col_by_preprocess_options?(col, options)
    return false unless options.kind_of?(Hash)
    return false unless options[:vim_performance_daily_adhoc]
    Metric::Rollup.excluded_col_for_expression?(col.to_sym)
  end

  def lenient_evaluate(obj, tz = nil)
    ruby_exp = to_ruby(tz)
    ruby_exp.nil? || Condition.subst_matches?(ruby_exp, obj)
  end

  def evaluate(obj, tz = nil)
    ruby_exp = to_ruby(tz)
    _log.debug("Expression before substitution: #{ruby_exp}")
    subst_expr = Condition.subst(ruby_exp, obj)
    _log.debug("Expression after substitution: #{subst_expr}")
    result = Condition.do_eval(subst_expr)
    _log.debug("Expression evaluation result: [#{result}]")
    result
  end

  def self.evaluate_atoms(exp, obj)
    exp = exp.kind_of?(self) ? copy_hash(exp.exp) : exp
    exp["result"] = new(exp).evaluate(obj)

    operators = exp.keys
    operators.each do|k|
      if ["and", "or"].include?(k.to_s.downcase)      # and/or atom is an array of atoms
        exp[k].each do|atom|
          evaluate_atoms(atom, obj)
        end
      elsif ["not", "!"].include?(k.to_s.downcase)    # not atom is a hash expression
        evaluate_atoms(exp[k], obj)
      else
        next
      end
    end
    exp
  end

  def self.operands2humanvalue(ops, options = {})
    # puts "Enter: operands2humanvalue: ops: #{ops.inspect}"
    ret = []
    if ops["tag"]
      v = nil
      ret.push(ops["alias"] || value2human(ops["tag"], options))
      MiqExpression.get_entry_details(ops["tag"]).each do|t|
        v = "'" + t.first + "'" if t.last == ops["value"]
      end
      if ops["value"] == :user_input
        v = "<user input>"
      else
        v ||= ops["value"].kind_of?(String) ? "'" + ops["value"] + "'" : ops["value"]
      end
      ret.push(v)
    elsif ops["field"]
      ops["value"] ||= ''
      if ops["field"] == "<count>"
        ret.push(nil)
        ret.push(ops["value"])
      else
        ret.push(ops["alias"] || value2human(ops["field"], options))
        if ops["value"] == :user_input
          ret.push("<user input>")
        else
          col_type = get_col_type(ops["field"]) || "string"
          ret.push(quote_human(ops["value"], col_type.to_s))
        end
      end
    elsif ops["count"]
      ret.push("COUNT OF " + (ops["alias"] || value2human(ops["count"], options)).strip)
      if ops["value"] == :user_input
        ret.push("<user input>")
      else
        ret.push(ops["value"])
      end
    elsif ops["regkey"]
      ops["value"] ||= ''
      ret.push(ops["regkey"] + " : " + ops["regval"])
      ret.push(ops["value"].kind_of?(String) ? "'" + ops["value"] + "'" : ops["value"])
    elsif ops["value"]
      ret.push(nil)
      ret.push(ops["value"])
    end
    ret
  end

  def self.value2human(val, options = {})
    options = {
      :include_model => true,
      :include_table => true
    }.merge(options)
    tables, col = val.split("-")
    first = true
    val_is_a_tag = false
    ret = ""
    if options[:include_table] == true
      friendly = tables.split(".").collect do|t|
        if t.downcase == "managed"
          val_is_a_tag = true
          "#{Tenant.root_tenant.name} Tags"
        elsif t.downcase == "user_tag"
          "My Tags"
        else
          if first
            first = nil
            next unless options[:include_model] == true
            Dictionary.gettext(t, :type => :model, :notfound => :titleize)
          else
            Dictionary.gettext(t, :type => :table, :notfound => :titleize)
          end
        end
      end.compact
      ret = friendly.join(".")
      ret << " : " unless ret.blank? || col.blank?
    end
    if val_is_a_tag
      if col
        classification = options[:classification] || Classification.find_by_name(col)
        ret << (classification ? classification.description : col)
      end
    else
      model = tables.blank? ? nil : tables.split(".").last.singularize.camelize
      dict_col = model.nil? ? col : [model, col].join(".")
      column_human = if col
                       if col.starts_with?(CustomAttributeMixin::CUSTOM_ATTRIBUTES_PREFIX)
                         col.gsub(CustomAttributeMixin::CUSTOM_ATTRIBUTES_PREFIX, "")
                       else
                         Dictionary.gettext(dict_col, :type => :column, :notfound => :titleize)
                       end
                     end
      ret << column_human if col
    end
    ret = " #{ret}" unless ret.include?(":")
    ret
  end

  def self.operands2rubyvalue(operator, ops, context_type)
    # puts "Enter: operands2rubyvalue: operator: #{operator}, ops: #{ops.inspect}"
    operator = operator.downcase
    ops["tag"] = ops["field"] if operator == "contains" && !ops["tag"] # process values in contains as tags

    if ops["tag"] && context_type != "hash"
      ref, val = value2tag(preprocess_managed_tag(ops["tag"]), ops["value"])
      ["<exist ref=#{ref}>#{val}</exist>"]
    elsif ops["tag"] && context_type == "hash"
      # This is only for supporting reporting "display filters"
      # In the report object the tag value is actually the description and not the raw tag name.
      # So we have to trick it by replacing the value with the description.
      description = MiqExpression.get_entry_details(ops["tag"]).inject("") do|s, t|
        break(t.first) if t.last == ops["value"]
        s
      end
      val = ops["tag"].split(".").last.split("-").join(".")
      fld = "<value type=string>#{val}</value>"
      [fld, quote(description, "string")]
    elsif ops["field"]
      if ops["field"] == "<count>"
        ["<count>", quote(ops["value"], "integer")]
      else
        col_type = get_col_type(ops["field"]) || "string"
        case context_type
        when "hash"
          val = ops["field"].split(".").last.split("-").join(".")
          fld = "<value type=#{col_type}>#{val}</value>"
        else
          ref, val = value2tag(ops["field"])
          fld = "<value ref=#{ref}, type=#{col_type}>#{val}</value>"
        end
        if ["like", "not like", "starts with", "ends with", "includes", "regular expression matches", "regular expression does not match"].include?(operator)
          [fld, ops["value"]]
        else
          [fld, quote(ops["value"], col_type.to_s)]
        end
      end
    elsif ops["count"]
      ref, count = value2tag(ops["count"])
      field = "<count ref=#{ref}>#{count}</count>"
      [field, quote(ops["value"], "integer")]
    elsif ops["regkey"]
      if operator == "key exists"
        "<registry key_exists=1, type=boolean>#{ops["regkey"].strip}</registry>  == 'true'"
      elsif operator == "value exists"
        "<registry value_exists=1, type=boolean>#{ops["regkey"].strip} : #{ops["regval"]}</registry>  == 'true'"
      else
        fld = "<registry>#{ops["regkey"].strip} : #{ops["regval"]}</registry>"
        if ["like", "not like", "starts with", "ends with", "includes", "regular expression matches", "regular expression does not match"].include?(operator)
          [fld, ops["value"]]
        else
          [fld, quote(ops["value"], "string")]
        end
      end
    end
  end
{'fraction_non_alphanumeric': 0.13354079696394686, 'fraction_numerical': 0.0026717267552182162, 'mean_word_length': 2.8625622984462034, 'pattern_counts': {'":': 2, '<': 93, '<?xml version=': 0, '>': 277, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6097298', 'n_tokens_mistral': 21227, 'n_tokens_neox': 20931, 'n_words': 4320}