<commit_before># Imports select snapshot data to daily_specs
# Does all snapshots in directory given

task :get_daily_specs => :environment do
  #require 'ruby-debug'
  write_instock_skus_into_file
  #analyze_daily_raw_specs
end

task :import_daily_attributes => :environment do
  # get historical data on raw product attributes data and write to daily specs
  raw = true
  import_data(raw)
end

task :import_daily_factors => :environment do
  # get historical factors data and write to daily specs
  raw = false
  import_data(raw)
end

def import_data(raw)
  #for local runs (change to own directory)
  directory = "/optemo/snapshots/slicehost"
  #for runs on jaguar
  #directory = "/mysql_backup/slicehost/linode"
  
  # loop over the files in the directory, unzipping gzipped files
  Dir.foreach(directory) do |entry|
    if entry =~ /\.gz/
      %x[gunzip #{directory}/#{entry}]
    end
  end
  # loop over each daily snapshot of the database (.sql file),
  # import it into the temp database, then get attributes for and write them to DailySpecs
  Dir.foreach(directory) do |snapshot|
    if snapshot =~ /\.sql/
      date = Date.parse(snapshot.chomp(File.extname(snapshot)))
      puts 'making records for date ' + date.to_s
      # import data from the snapshot to the temp database
      puts "mysql -u optemo -p ***REMOVED*** -h jaguar temp < #{directory}/#{snapshot}"
      %x[mysql -u optemo -p***REMOVED*** -h jaguar temp < #{directory}/#{snapshot}]
      #username and password cannot be company's (optemo, tiny******) - Must be local users
      ActiveRecord::Base.establish_connection(:adapter => "mysql2", :database => "temp", :host => "jaguar",
        :username => "marc", :password => "<PASSWORD>")
      case raw
      when true
        specs = get_instock_attributes()
      when false
        specs = get_instock_factors()
      end
      ActiveRecord::Base.establish_connection(:development)
      update_daily_specs(date, specs, raw)
    end
  end
end

# collects factor values for instock cameras
# assumes an active connection to the temp database 
# output: an array of hashes of the selected factors, one entry per product
def get_instock_factors()
  
  specs = []
  
  #instock = Product.find_all_by_product_type_and_instock("camera_bestbuy", 1)
  
  instock.each do |p|
    
    sku = p.sku
    pid = p.id
    price_factor = ContSpec.find_by_product_id_and_name(pid,"price_factor")
    maxresolution_factor = ContSpec.find_by_product_id_and_name(pid,"maxresolution_factor")
    opticalzoom_factor = ContSpec.find_by_product_id_and_name(pid,"opticalzoom_factor")
    
    # if any of the expected features do not have values in the DB, don't create a new spec for that product
    if maxresolution_factor.nil? or opticalzoom_factor.nil? or price_factor.nil?<commit_msg>Setup import_snapshot_data for running on jaguar<commit_after># Imports select snapshot data to daily_specs
# Does all snapshots in directory given

task :get_daily_specs => :environment do
  #require 'ruby-debug'
  write_instock_skus_into_file
  #analyze_daily_raw_specs
end

task :import_daily_attributes => :environment do
  # get historical data on raw product attributes data and write to daily specs
  raw = true
  import_data(raw)
end

task :import_daily_factors => :environment do
  # get historical factors data and write to daily specs
  raw = false
  import_data(raw)
end

def import_data(raw)
  #for local runs (change to own directory)
  #directory = "/optemo/snapshots/slicehost"
  #for runs on jaguar
  directory = "/mysql_backup/slicehost/linode"
  
  # loop over the files in the directory, unzipping gzipped files
  Dir.foreach(directory) do |entry|
    if entry =~ /\.gz/
      %x[gunzip #{directory}/#{entry}]
    end
  end
  # loop over each daily snapshot of the database (.sql file),
  # import it into the temp database, then get attributes for and write them to DailySpecs
  Dir.foreach(directory) do |snapshot|
    if snapshot =~ /\.sql/
      date = Date.parse(snapshot.chomp(File.extname(snapshot)))
      puts 'making records for date ' + date.to_s
      # import data from the snapshot to the temp database
      puts "mysql -u optemo -p ***REMOVED*** -h jaguar temp < #{directory}/#{snapshot}"
      %x[mysql -u optemo -p***REMOVED*** -h jaguar temp < #{directory}/#{snapshot}]
      #username and password cannot be company's (optemo, tiny******) - Must be local user's if run locally
      ActiveRecord::Base.establish_connection(:adapter => "mysql2", :database => "temp", :host => "jaguar",
        :username => "optemo", :password => "***<PASSWORD>***")
      case raw
      when true
        specs = get_instock_attributes()
      when false
        specs = get_instock_factors()
      end
      ActiveRecord::Base.establish_connection(:development)
      update_daily_specs(date, specs, raw)
    end
  end
end

# collects factor values for instock cameras
# assumes an active connection to the temp database 
# output: an array of hashes of the selected factors, one entry per product
def get_instock_factors()
  
  specs = []
  
  #instock = Product.find_all_by_product_type_and_instock("camera_bestbuy", 1)
  
  instock.each do |p|
    
    sku = p.sku
    pid = p.id
    price_factor = ContSpec.find_by_product_id_and_name(pid,"price_factor")
    maxresolution_factor = ContSpec.find_by_product_id_and_name(pid,"maxresolution_factor")
    opticalzoom_factor = ContSpec.find_by_product_id_and_name(pid,"opticalzoom_factor")
    
    # if any of the expected features do not have values in the DB, don't create a new spec for that product
    if maxresolution_factor.nil? or opticalzoom_factor.nil? or price_factor.nil?
{'fraction_non_alphanumeric': 0.09425287356321839, 'fraction_numerical': 0.0007073386383731211, 'mean_word_length': 3.7891617273497036, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3611229', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1728, 'n_words': 561}