<commit_before>#Here is where general upkeep scripts are
desc "Calculate factors for all features of all products, and pre-calculate utility scores"
task :calculate_factors => :environment do
  # Do not truncate Factor table anymore. Instead, add more factors for the given URL.
  file = YAML::load(File.open("#{Rails.root}/config/products.yml"))
  unless ENV.include?("url") && (s = Session.new(ENV["url"])) && file[ENV["url"]] && (!ENV.include?("version") || (ENV.include?("version") && ENV["version"].to_i.to_s == ENV["version"] && ENV["version"].to_i >= 0))
    raise "usage: rake calculate_factors url=? (version=?) # url is a valid url from products.yml; sets product_type. `version` is optional; if not supplied, the version number will be incremented based on the maximum existing factor version for that product_type."
  end
  
  cont_activerecords = []
  #cat_activerecords =[]
  #bin_activerecords = []
  cont_spec_local_cache = {} # This saves doing many ContSpec lookups. It's a hash with {id => value} pairs
  all_products = Product.valid.instock
  all_products.each do |product|
    utility = []
    s.continuous["filter"].each do |f|
      unless cont_spec_local_cache[f]
        records = ContSpec.find(:all, :select => 'product_id, value', :conditions => ["product_id IN (?) and name = ?", all_products, f])
        temp_hash = {}
        records.each do |r| # Strip the records down to {id => value} pairs
          temp_hash[r.product_id] = r.value
        end
        cont_spec_local_cache[f] = temp_hash
      end
      newFactorRow = ContSpec.new({:product_id => product.id, :product_type => s.product_type, :name => f+"_factor"})
      fVal = cont_spec_local_cache[f][product.id]
      debugger unless fVal # The alternative here is to crash. This should never happen if Product.valid.instock is doing its job.
      newFactorRow.value = calculateFactor(fVal, f, cont_spec_local_cache[f])
      utility << newFactorRow.value
      cont_activerecords.push(newFactorRow)
    end
    #Add the static calculated utility
    cont_activerecords.push ContSpec.new({:product_id => product.id, :product_type => s.product_type, :name => "utility", :value => utility.sum})
  end
  
  ContSpec.delete_all(["name = 'utility' and product_type = ?", s.product_type])
  s.continuous["filter"].each{|f| ContSpec.delete_all(["name = ? and product_type = ?", f+"_factor", s.product_type])} # ContSpec records do not have a version number, so we have to wipe out the old ones.  
  # Do all record saving at the end for efficiency
  ContSpec.transaction do
    cont_activerecords.each(&:save)
  end   
end

desc "Run boostexter to generate strong hypothesis files or Parse boostexter strong hypothesis files and save in database"
task :btxtr => :environment do
     $: << "#{Rails.root}/lib/cluster_labeling/boostexter_labels_rb"
     
     unless ENV.include?("url") && ENV.include?("action") && (ENV['action']=='save' || ENV['action']=='train')
          raise "usage: rake btxtr url=? action=? # url is a valid url from products.yml and action is 'save' or 'train'" 
     end
     
     s = Session.new(ENV['url'])
     s.version = Cluster.maximum(:version, :conditions => ['product_type = ?', s.product_type]) if s.directLayout
     case ENV['action']
     when 'train'
       require 'train_boostexter.rb'
       BtxtrLabels.train_boostexter_on_all_clusters()
     when 'save'
       require 'combined_rules.rb'
       BtxtrLabels.save_combined_rules_for_all_clusters()
     end
end

desc "Assign lowest price to printers"
task :assign_lowest_price => :environment do
  Printer.find(:all, :conditions => ['created_at > ?', 4.days.ago]).each do |p|
    #Find lowest product price
    os = RetailerOffering.find_all_by_product_id_and_product_type(p.id,p.class.name)
    lowest = 1000000000
    p.instock = false
    if !os.nil? && !os.empty?
      os.each do |o| 
        if o.stock && o.priceint && o.priceint < lowest
          lowest = o.priceint
          p.price = lowest
          p.pricestr = o.pricestr
          p.bestoffer = o.id
          p.instock = true
        end
      end
    end
    p.save
  end
end

def calculateFactor(fVal, f, contspecs)
  # Order the feature values, reversed to give the highest value to duplicates
  ordered = contspecs.values.sort
  ordered = ordered.reverse if Session.current.prefDirection[f] == 1
  return 0 if Session.current.prefDirection[f] == 0
  pos = ordered.index(fVal)
  len = ordered.length
  (len - pos)/len.to_f
end

desc "Run c-code to recluster"
task :c_clustering do
  require 'helpers/clusteringLogCheckLinux'
  include Clusteringlogchecklinux
  env = ENV['RAILS_ENV'] || 'development'
  ['printer', 'camera'].each do |prodtype|
    ['us','ca'].each do |region|
      `#{Rails.root}/lib/c_code/clusteringCodes/codes/hCluster #{prodtype} #{region} #{env} #{Session.current.numGroups}`
      cleanupInvalidDatabase(prodtype)
    end
  end
end

desc "Recluster printers"
task :recluster => [:calculate_factors,:c_clustering] do
  Rake::Task['db:properties'].invoke
end

desc 'Create YAML test fixtures from data in an existing database.  
Defaults to development database.  Set RAILS_ENV to override.'

task :extract_fixtures => :environment do
  #sql  = "SELECT * FROM %s where (product_type = 'printer_us' and version = 12) or product_type = 'camera_us'"
  sql = "SELECT * FROM %s"
  ActiveRecord::Base.establish_connection
  table_name = ENV["T"]
    i = "000"
    File.open("#{Rails.root}/test/fixtures/#{table_name}.yml", 'w') do |file|
      data = ActiveRecord::Base.connection.select_all(sql % table_name)
      myhash = {}
      file.write data[0..20000].inject(myhash) { |hash, record|
        hash["#{table_name}_#{i.succ!}"] = record
        hash
      }.to_yaml<commit_msg>Moved calculate factors to the firehose project<commit_after>desc "Run boostexter to generate strong hypothesis files or Parse boostexter strong hypothesis files and save in database"
task :btxtr => :environment do
     $: << "#{Rails.root}/lib/cluster_labeling/boostexter_labels_rb"
     
     unless ENV.include?("url") && ENV.include?("action") && (ENV['action']=='save' || ENV['action']=='train')
          raise "usage: rake btxtr url=? action=? # url is a valid url from products.yml and action is 'save' or 'train'" 
     end
     
     s = Session.new(ENV['url'])
     s.version = Cluster.maximum(:version, :conditions => ['product_type = ?', s.product_type]) if s.directLayout
     case ENV['action']
     when 'train'
       require 'train_boostexter.rb'
       BtxtrLabels.train_boostexter_on_all_clusters()
     when 'save'
       require 'combined_rules.rb'
       BtxtrLabels.save_combined_rules_for_all_clusters()
     end
end

desc 'Create YAML test fixtures from data in an existing database.  
Defaults to development database.  Set RAILS_ENV to override.'

task :extract_fixtures => :environment do
  #sql  = "SELECT * FROM %s where (product_type = 'printer_us' and version = 12) or product_type = 'camera_us'"
  sql = "SELECT * FROM %s"
  ActiveRecord::Base.establish_connection
  table_name = ENV["T"]
    i = "000"
    File.open("#{Rails.root}/test/fixtures/#{table_name}.yml", 'w') do |file|
      data = ActiveRecord::Base.connection.select_all(sql % table_name)
      myhash = {}
      file.write data[0..20000].inject(myhash) { |hash, record|
        hash["#{table_name}_#{i.succ!}"] = record
        hash
      }.to_yaml
{'fraction_non_alphanumeric': 0.1281947261663286, 'fraction_numerical': 0.0050033806626098715, 'mean_word_length': 3.6398996235884566, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '76373', 'n_tokens_mistral': 2434, 'n_tokens_neox': 2345, 'n_words': 637}