<commit_before>class Search < ActiveRecord::Base
  require 'set'
  include CachingMemcached
  belongs_to :session
  
  ## Computes distributions (arrays of normalized product counts) for all continuous features 
  def distribution(featureName)
       dist = Array.new(21,0)
       min = DbFeature.featurecache(featureName).min
       max = DbFeature.featurecache(featureName).max
       stepsize = (max-min) / dist.length + 0.000001 #Offset prevents overflow of 10 into dist array
       itof = $model::ItoF.include?(featureName)
       acceptedNodes.each do |n|
         if (itof==true && min>=max-1) || (itof==false && min>=max-0.1)
           #No range so fill everything in
           dist = Array.new(dist.length,1)
         else
           i = ((n.send(featureName) - min) / stepsize).to_i
           dist[i] += 1 if i < dist.length
         end
       end  
       round2Decim(normalize(dist))
  end
  
  def countBinary(featureName)
     acceptedNodes.map{|n| n[featureName]? 1 : 0}.sum
  end
  
  def countBrands(brandName)
    
    acceptedNodes.map{|n| (n["brand"]== brandName)? 1 : 0}.sum
  end
  
  def acceptedNodes
    @acceptedNodes ||= clusters.map{|c| c.nodes}.flatten 
  end
  
  #Range of product offerings
  def ranges(featureName)
     @sRange ||= {}
     if @sRange[featureName].nil?
       min = clusters.map{|c|c.ranges(featureName)[0]}.compact.sort[0]
       max = clusters.map{|c|c.ranges(featureName)[1]}.compact.sort[-1] 
       @sRange[featureName] = [min, max]
     end
     @sRange[featureName]  
  end
  
  def indicator(featureName)
    indic = false
    values = clusters.map{|c| c.indicator(featureName)}
    if values.index(false).nil?
      indic = true
    end  
    indic
  end
  
  def relativeDescriptions
    return if clusters.empty?
    @reldescs ||= []
    if @reldescs.empty?
      feats = {}
      $model::ContinuousFeaturesF.each do |f|
        norm = DbFeature.featurecache(f).max - DbFeature.featurecache(f).min
        norm = 1 if norm == 0
        feats[f] = clusters.map{ |c| c.representative}.compact.map {|c| c[f].to_f/norm } 
      end
      cluster_count.times do |i|
        dist = {}
        $model::ContinuousFeaturesF.each do |f|
          if feats[f].min == feats[f][i]
            dist[f] = feats[f].sort[1] - feats[f][i]
            dist.delete(f) if dist[f] == 0 && feats[f].sort[2] == feats[f][i]#Remove ties
          elsif feats[f].max == feats[f][i]
            dist[f] = feats[f][i] - feats[f].sort[-2]
            dist.delete(f) if dist[f] == 0  && feats[f].sort[-3] == feats[f][i]#Remove ties
          elsif feats[f].sort[4] == feats[f][i]
            dist[f] = ([feats[f].sort[4]-feats[f].sort[3],feats[f].sort[5]-feats[f].sort[4]].min) - 1
            dist.delete(f) if dist[f] == -1 #Remove ties
          end
        end if cluster_count > 1
        n = dist.count
        d = []
        dist.sort{|a,b| b[1] <=> a[1]}[0..1].each do |f,v|
          dir = feats[f].min == feats[f][i] ? "lower" : "higher"
          dir = feats[f].min == feats[f][i] ? "lower" : feats[f].max == feats[f][i] ? "higher" : "avg"
          d << dir+f
        end
        #Add binary labels
        #d.unshift "Waterproof" if clusters[i].waterproof && layer == 1
        #d.unshift "SLR" if clusters[i].slr && layer == 1
        if d.empty?
          @reldescs << ["avg"]
        else
          @reldescs << d
        end
        #@descs[-1] = @descs.last + " (#{n})"
      end
    end
    @reldescs
  end
    
  def boostexterClusterDescriptions
    if @returned_taglines 
      return @returned_taglines
    else
      @returned_taglines = []
    end
    weighted_averages = {}
    return if clusters.empty?
    descriptions = Array.new
    featureNameSet = Set.new
    clusters.each do |c|
      weighted_averages[c.id] = {}
      current_nodes = c.nodes
      product_ids = current_nodes.map {|n| n.product_id }

      rules = BoostexterCombinedRule.find(:all, :order => "weight DESC", :conditions => {"cluster_id" => c.id, "version" => Session.current.version})
      
      unless product_ids.empty?
        rules.each do |r|
          unpacked_weighted_intervals = YAML.load(r.yaml_repr).map {|i| [i["interval"], i["weight"]]}
          z = 0
          weighted_average = 0
          # The products hash has all the cache hits for cameras serialized as a single request.
          #products = c.findCachedProducts(product_ids)
          product_ids.each do |id| 
            # This is out of cachingMemcached.
            #product = products["#{$model}#{Session.current.version}#{id}"]
            # Caching is done using @@products, as with db_features, etc.
            product = $model.productcache(id)
            feature_value = product.send(r.fieldname)
            next unless feature_value
            weight = find_weight_for_value(unpacked_weighted_intervals, feature_value)
            weighted_average += weight * feature_value
            z += weight
          end
          next if z == 0 # Loop back to the beginning; do not add this field name for this cluster
          weighted_average /= z
          weighted_averages[c.id][r.fieldname] = weighted_average
          featureNameSet.add(r.fieldname)
        end
      end
    end
    weighted_averages.each do |cluster_id,featurehash|
      current_cluster_tagline = []
      featurehash.each do |featurename,weighted_average|
        quartiles = compute_quartile(featurename)
        if weighted_average < quartiles[0].to_f # This is low for the given feature
          current_cluster_tagline.push("lower#{featurename}")
        elsif weighted_average > quartiles[1].to_f # This is high for the given feature
          current_cluster_tagline.push("higher#{featurename}")
        else # Inclusion. It's between 25% and 50%
          # This is boring. Do nothing.
          current_cluster_tagline.push("avg#{featurename}")
        end
        break if current_cluster_tagline.length == 2
      end
      @returned_taglines.push(current_cluster_tagline)
    end
    return @returned_taglines # [ ["avgdisplaysize", "highminimumfocallength"],["avgprice", ""] , ...] 
  end

def compute_quartile(featurename)
  # This can be sped up by the following: Instead of fetching p.maximumresolution, then p.displaysize, etc.,
  # just do a single query for p. If there are multiple features to fetch, the rest of the query is guaranteed to be identical
  # and doing activerecord caching will help
  filter_query_thing = ""
  filter_query_thing = Cluster.filterquery(Session.current, 'n.') + " AND " if Session.current.filter && !Cluster.filterquery(Session.current, 'n.').blank?
  cluster_ids = clusters.map{|c| c.id}.join(", ")
  product_count = ActiveRecord::Base.connection.select_one("select count(distinct(p.id)) from #{$model.table_name} p, #{$nodemodel.table_name} n, #{$clustermodel.table_name} cc WHERE p.#{featurename} is not NULL AND #{filter_query_thing} n.product_id = p.id AND cc.id = n.cluster_id AND cc.id IN (#{cluster_ids})")
  product_count = product_count["count(distinct(p.id))"].to_i
  q25offset = (product_count / 4.0).floor
  q75offset = ((product_count * 3) / 4.0).floor
  q25 = ActiveRecord::Base.connection.select_one("select p.#{featurename} from #{$model.table_name} p, #{$nodemodel.table_name} n, #{$clustermodel.table_name} cc WHERE p.#{featurename} is not NULL AND #{filter_query_thing} n.product_id = p.id AND cc.id = n.cluster_id AND cc.id IN (#{cluster_ids}) ORDER BY #{featurename} LIMIT 1 OFFSET #{q25offset}")
  q75 = ActiveRecord::Base.connection.select_one("select p.#{featurename} from #{$model.table_name} p, #{$nodemodel.table_name} n, #{$clustermodel.table_name} cc WHERE p.#{featurename} is not NULL AND #{filter_query_thing} n.product_id = p.id AND cc.id = n.cluster_id AND cc.id IN (#{cluster_ids}) ORDER BY #{featurename} LIMIT 1 OFFSET #{q75offset}")

  [q25[featurename], q75[featurename]]
end
  
  def clusterDescription(clusterNumber)
    return if clusters.empty?
    clusterDs = Array.new 
    des = []
    slr=0
      $model::BinaryFeatures.each do |f|<commit_msg>Optimized non-cached version somewhat<commit_after>class Search < ActiveRecord::Base
  require 'set'
  include CachingMemcached
  belongs_to :session
  
  ## Computes distributions (arrays of normalized product counts) for all continuous features 
  def distribution(featureName)
       dist = Array.new(21,0)
       min = DbFeature.featurecache(featureName).min
       max = DbFeature.featurecache(featureName).max
       stepsize = (max-min) / dist.length + 0.000001 #Offset prevents overflow of 10 into dist array
       itof = $model::ItoF.include?(featureName)
       acceptedNodes.each do |n|
         if (itof==true && min>=max-1) || (itof==false && min>=max-0.1)
           #No range so fill everything in
           dist = Array.new(dist.length,1)
         else
           i = ((n.send(featureName) - min) / stepsize).to_i
           dist[i] += 1 if i < dist.length
         end
       end  
       round2Decim(normalize(dist))
  end
  
  def countBinary(featureName)
     acceptedNodes.map{|n| n[featureName]? 1 : 0}.sum
  end
  
  def countBrands(brandName)
    
    acceptedNodes.map{|n| (n["brand"]== brandName)? 1 : 0}.sum
  end
  
  def acceptedNodes
    @acceptedNodes ||= clusters.map{|c| c.nodes}.flatten 
  end
  
  #Range of product offerings
  def ranges(featureName)
     @sRange ||= {}
     if @sRange[featureName].nil?
       min = clusters.map{|c|c.ranges(featureName)[0]}.compact.sort[0]
       max = clusters.map{|c|c.ranges(featureName)[1]}.compact.sort[-1] 
       @sRange[featureName] = [min, max]
     end
     @sRange[featureName]  
  end
  
  def indicator(featureName)
    indic = false
    values = clusters.map{|c| c.indicator(featureName)}
    if values.index(false).nil?
      indic = true
    end  
    indic
  end
  
  def relativeDescriptions
    return if clusters.empty?
    @reldescs ||= []
    if @reldescs.empty?
      feats = {}
      $model::ContinuousFeaturesF.each do |f|
        norm = DbFeature.featurecache(f).max - DbFeature.featurecache(f).min
        norm = 1 if norm == 0
        feats[f] = clusters.map{ |c| c.representative}.compact.map {|c| c[f].to_f/norm } 
      end
      cluster_count.times do |i|
        dist = {}
        $model::ContinuousFeaturesF.each do |f|
          if feats[f].min == feats[f][i]
            dist[f] = feats[f].sort[1] - feats[f][i]
            dist.delete(f) if dist[f] == 0 && feats[f].sort[2] == feats[f][i]#Remove ties
          elsif feats[f].max == feats[f][i]
            dist[f] = feats[f][i] - feats[f].sort[-2]
            dist.delete(f) if dist[f] == 0  && feats[f].sort[-3] == feats[f][i]#Remove ties
          elsif feats[f].sort[4] == feats[f][i]
            dist[f] = ([feats[f].sort[4]-feats[f].sort[3],feats[f].sort[5]-feats[f].sort[4]].min) - 1
            dist.delete(f) if dist[f] == -1 #Remove ties
          end
        end if cluster_count > 1
        n = dist.count
        d = []
        dist.sort{|a,b| b[1] <=> a[1]}[0..1].each do |f,v|
          dir = feats[f].min == feats[f][i] ? "lower" : "higher"
          dir = feats[f].min == feats[f][i] ? "lower" : feats[f].max == feats[f][i] ? "higher" : "avg"
          d << dir+f
        end
        #Add binary labels
        #d.unshift "Waterproof" if clusters[i].waterproof && layer == 1
        #d.unshift "SLR" if clusters[i].slr && layer == 1
        if d.empty?
          @reldescs << ["avg"]
        else
          @reldescs << d
        end
        #@descs[-1] = @descs.last + " (#{n})"
      end
    end
    @reldescs
  end
    
  def boostexterClusterDescriptions
    if @returned_taglines 
      return @returned_taglines
    else
      @returned_taglines = []
    end
    weighted_averages = {}
    return if clusters.empty?
    descriptions = Array.new
    featureNameSet = Set.new
    clusters.each do |c|
      weighted_averages[c.id] = {}
      current_nodes = c.nodes
      product_ids = current_nodes.map {|n| n.product_id }
      product_query_string = "id IN (" + product_ids.join(" OR ") + ")"

      rules = BoostexterCombinedRule.find(:all, :order => "weight DESC", :conditions => {"cluster_id" => c.id, "version" => Session.current.version})
      unless product_ids.empty?
        # This is going to be hard to cache. There needs to be a hash created out of a string concatenation of all the product_ids maybe? Think about this one
        @products = $model.find(:all, :conditions => {:id => product_ids }).index_by(&:id)
        rules.each do |r|
          unpacked_weighted_intervals = YAML.load(r.yaml_repr).map {|i| [i["interval"], i["weight"]]}
          z = 0
          weighted_average = 0
          # The products hash has all the cache hits for cameras serialized as a single request.
          #products = c.findCachedProducts(product_ids)
          product_ids.each do |id| 
            # This is out of cachingMemcached.
            #product = products["#{$model}#{Session.current.version}#{id}"]
            # This line here should no longer be needed # products[id] = $model.productcache(id) unless (products[id])            
            product = @products[id]
            feature_value = product.send(r.fieldname)
            next unless feature_value
            weight = find_weight_for_value(unpacked_weighted_intervals, feature_value)
            weighted_average += weight * feature_value
            z += weight
          end
          next if z == 0 # Loop back to the beginning; do not add this field name for this cluster.
          weighted_average /= z
          weighted_averages[c.id][r.fieldname] = weighted_average
          featureNameSet.add(r.fieldname)
        end
      end
    end
    weighted_averages.each do |cluster_id,featurehash|
      current_cluster_tagline = []
      featurehash.each do |featurename,weighted_average|
        quartiles = compute_quartile(featurename)
        if weighted_average < quartiles[0].to_f # This is low for the given feature
          current_cluster_tagline.push("lower#{featurename}")
        elsif weighted_average > quartiles[1].to_f # This is high for the given feature
          current_cluster_tagline.push("higher#{featurename}")
        else # Inclusion. It's between 25% and 75%
          # Do nothing? Averages are included for now, comment this out to remove
          current_cluster_tagline.push("avg#{featurename}")
        end
        break if current_cluster_tagline.length == 2 # Limit to 2 taglines per cluster (this is due to a space limitation in the UI)
      end
      @returned_taglines.push(current_cluster_tagline)
    end
    return @returned_taglines # [ ["avgdisplaysize", "highminimumfocallength"],["avgprice", ""] , ...] 
  end

def compute_quartile(featurename)
  # This can be sped up by the following: Instead of fetching p.maximumresolution, then p.displaysize, etc.,
  # just do a single query for p. If there are multiple features to fetch, the rest of the query is guaranteed to be identical
  # and doing activerecord caching will help
  filter_query_thing = ""
  filter_query_thing = Cluster.filterquery(Session.current, 'n.') + " AND " if Session.current.filter && !Cluster.filterquery(Session.current, 'n.').blank?
  cluster_ids = clusters.map{|c| c.id}.join(", ")
  product_count = ActiveRecord::Base.connection.select_one("select count(distinct(p.id)) from #{$model.table_name} p, #{$nodemodel.table_name} n, #{$clustermodel.table_name} cc WHERE p.#{featurename} is not NULL AND #{filter_query_thing} n.product_id = p.id AND cc.id = n.cluster_id AND cc.id IN (#{cluster_ids})")
  product_count = product_count["count(distinct(p.id))"].to_i
  q25offset = (product_count / 4.0).floor
  q75offset = ((product_count * 3) / 4.0).floor
  # Although we have @products, the database *should* be substantially faster at sorting them for each quartile computation.
  # However, we might be limited by the network connection here instead. For database connections on localhost, probably not, so leave as-is.
  q25 = ActiveRecord::Base.connection.select_one("select p.#{featurename} from #{$model.table_name} p, #{$nodemodel.table_name} n, #{$clustermodel.table_name} cc WHERE p.#{featurename} is not NULL AND #{filter_query_thing} n.product_id = p.id AND cc.id = n.cluster_id AND cc.id IN (#{cluster_ids}) ORDER BY #{featurename} LIMIT 1 OFFSET #{q25offset}")
  q75 = ActiveRecord::Base.connection.select_one("select p.#{featurename} from #{$model.table_name} p, #{$nodemodel.table_name} n, #{$clustermodel.table_name} cc WHERE p.#{featurename} is not NULL AND #{filter_query_thing} n.product_id = p.id AND cc.id = n.cluster_id AND cc.id IN (#{cluster_ids}) ORDER BY #{featurename} LIMIT 1 OFFSET #{q75offset}")
  [q25[featurename], q75[featurename]]
end
  
  def clusterDescription(clusterNumber)
    return if clusters.empty?
    clusterDs = Array.new 
    des = []
    slr=0
      $model::BinaryFeatures.each do |f|
{'fraction_non_alphanumeric': 0.1292351510116893, 'fraction_numerical': 0.009909214976562036, 'mean_word_length': 2.8079530049706283, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '310948', 'n_tokens_mistral': 5652, 'n_tokens_neox': 5502, 'n_words': 1342}