<commit_before>#Here is where general upkeep scripts are
desc "This is for testing only"
task :upkeep => :environment do
  if !ENV.include?("product_type")
    Session.new
  else
    if /[BF]\w+/ =~ ENV["product_type"]
      Session.new ENV["product_type"]
    else
      raise "usage: rake update product_type=? # where product_type is a Bestbuy hierarchy number, e.g. B20218 or F1084."
    end
  end
  #!!!!!! For testing only
#  #Calculate new spec factors
#  Product.calculate_factors
# #Get the color relationships loaded
    ProductSibling.get_relations
end

desc "Update data automatically"
task :update => :environment do
  
  if !ENV.include?("product_type")
    Session.new
  else
    if /[BF]\w+/ =~ ENV["product_type"]
      Session.new ENV["product_type"]
    else
      raise "usage: rake update product_type=? # where product_type is a Bestbuy hierarchy number, e.g. B20218 or F1084."
    end
  end
  
  start = Time.now
  leaves = Session.product_type_leaves
  if leaves.nil? || leaves.empty?
    raise "Product type: #{ENV["product_type"]} not found"
  end

  leaves.each do |node|
    #Run the update task for this leaf node
    Session.new node
    begin 
      print 'Started scraping category ' + node.to_s
      Product.feed_update
      puts '...Finished'
    rescue BestBuyApi::RequestError => error
      puts 'Got the following error in scraping current category: '
      puts error.to_s
    end
  end
  Session.new ENV["product_type"] #Reset session
  #clean up inactive scraping rules not used any more
  Facet.check_active
  Search.cleanup_history_data(7)
  #Report problem with script if it finishes too fast
  `touch tmp/r_updateproblem.txt` if (Time.now - start < 1.minute)
end

# This task takes a product category or list of product categories as an argument.
# It determines the leaf nodes for these categories, and spawns subprocesses which 
# execute the update_leaf task for each of these leaves in parallel.
#
# Use commas to separate multiple product categories, for example:
#   rake update_parallel product_type=B20218,B29157,F1002
desc "Update product data in parallel"
task :update_parallel => :environment do
  DEFAULT_CATEGORY = "B20218"

  product_types = [DEFAULT_CATEGORY]
  
  if ENV.include?("product_type")
    product_types = ENV["product_type"].split(/,/)
  else 
    puts "update_parallel: No product_type command-line argument, using default category " + DEFAULT_CATEGORY
  end
  puts "update_parallel started at " + Time.now.to_s + ", categories = " + product_types.to_s
  
  leaves = []
  product_types.each do |type|
    some_leaves = ProductCategory.get_leaves(type)
    if some_leaves.nil? || some_leaves.empty?
      puts "No leaves found for category " + type
    else
      leaves.concat(some_leaves)
    end
  end 
  
  if leaves.empty?
    raise "No leaf nodes found for specified categories"
  end

  puts 'Found leaf nodes: ' + leaves.to_s

  start = Time.now
  # We first obtain the product list for each leaf category in series. Doing this step in parallel
  # seems to result in strange behavior for Best Buy's search API call.
  #
  # We store the product list for each leaf category in a temporary file and pass the path 
  # to the temp file to the child process responsible for scraping the category.
  #
  # We store temp file references in an array, to ensure they do not get garbage collected before 
  # child processes have a chance to read them (the temp files are deleted when they are garbage collected).
  temp_files = []
  curr_child_process_count = 0
  spawned_processes = 0
  leaves.each do |node|
    Session.new node
    products = Product.get_products(node)

    # Save product information to a temporary file
    temp_file = Tempfile.open("category_products", Rails.root.join('tmp')) { |temp_file|
      YAML.dump(products, temp_file)
      temp_file
    }
    temp_files << temp_file

    if curr_child_process_count >= 10 
      Process.wait
      curr_child_process_count -= 1
    end
    command_line = "bundle exec rake update_leaf product_type=#{node} file=#{temp_file.path}"
    if Rake.application.options.trace
      command_line += " --trace"
    end
    Process.spawn(command_line)
    curr_child_process_count += 1
    spawned_processes += 1
    puts "Spawned rake task for node " + node + " (" + spawned_processes.to_s + "/" + leaves.size.to_s + ")"
  end
                  
  puts "Finished spawning child processes"

  # Wait for all child processes to finish
  Process.waitall

  product_types.each do |product_type|
    Session.new product_type #Reset session
    #clean up inactive scraping rules not used any more
    Facet.check_active
  end

  Search.cleanup_history_data(7)

  #Report problem with script if it finishes too fast
  `touch tmp/r_updateproblem.txt` if (Time.now - start < 1.minute)

  puts "update_parallel finished at " + Time.now.to_s
end

# This task is not intended to be invoked directly. Instead it is invoked by 
# update_parallel for each leaf node under a category.
desc "Update data for a single leaf node"
task :update_leaf => :environment do
  
  unless ENV.include?("product_type") && ENV.include?("file")
    raise "usage: rake update_leaf product_type=? file=?"
  end
  
  node = ENV["product_type"]
  file_path = ENV["file"]

  # Force BBproduct class to be auto-loaded.
  BBproduct.new
  products = YAML.load_file(file_path)

  Session.new node
  begin 
    puts "update_leaf scraping " + products.size.to_s + " products for category " + node
    Product.feed_update(products)
    puts 'update_leaf finished scraping category ' + node
  rescue BestBuyApi::RequestError => error
    puts 'update_leaf got the following error in scraping category ' + node + ': '
    puts error.to_s
  end
end

namespace :cache do
  desc 'Clear memcache'
  task :clear => :environment do<commit_msg>Changed some 'puts' calls in update task to Rails.logger.info calls.<commit_after>#Here is where general upkeep scripts are
desc "This is for testing only"
task :upkeep => :environment do
  if !ENV.include?("product_type")
    Session.new
  else
    if /[BF]\w+/ =~ ENV["product_type"]
      Session.new ENV["product_type"]
    else
      raise "usage: rake update product_type=? # where product_type is a Bestbuy hierarchy number, e.g. B20218 or F1084."
    end
  end
  #!!!!!! For testing only
#  #Calculate new spec factors
#  Product.calculate_factors
# #Get the color relationships loaded
    ProductSibling.get_relations
end

desc "Update data automatically"
task :update => :environment do
  
  if !ENV.include?("product_type")
    Session.new
  else
    if /[BF]\w+/ =~ ENV["product_type"]
      Session.new ENV["product_type"]
    else
      raise "usage: rake update product_type=? # where product_type is a Bestbuy hierarchy number, e.g. B20218 or F1084."
    end
  end
  
  start = Time.now
  leaves = Session.product_type_leaves
  if leaves.nil? || leaves.empty?
    raise "Product type: #{ENV["product_type"]} not found"
  end

  leaves.each do |node|
    #Run the update task for this leaf node
    Session.new node
    begin 
      print 'Started scraping category ' + node.to_s
      Product.feed_update
      puts '...Finished'
    rescue BestBuyApi::RequestError => error
      puts 'Got the following error in scraping current category: '
      puts error.to_s
    end
  end
  Session.new ENV["product_type"] #Reset session
  #clean up inactive scraping rules not used any more
  Facet.check_active
  Search.cleanup_history_data(7)
  #Report problem with script if it finishes too fast
  `touch tmp/r_updateproblem.txt` if (Time.now - start < 1.minute)
end

# This task takes a product category or list of product categories as an argument.
# It determines the leaf nodes for these categories, and spawns subprocesses which 
# execute the update_leaf task for each of these leaves in parallel.
#
# Use commas to separate multiple product categories, for example:
#   rake update_parallel product_type=B20218,B29157,F1002
desc "Update product data in parallel"
task :update_parallel => :environment do
  DEFAULT_CATEGORY = "B20218"

  product_types = [DEFAULT_CATEGORY]
  
  if ENV.include?("product_type")
    product_types = ENV["product_type"].split(/,/)
  else 
    puts "update_parallel: No product_type command-line argument, using default category " + DEFAULT_CATEGORY
  end
  puts "update_parallel started at " + Time.now.to_s + ", categories = " + product_types.to_s
  
  leaves = []
  product_types.each do |type|
    some_leaves = ProductCategory.get_leaves(type)
    if some_leaves.nil? || some_leaves.empty?
      puts "No leaves found for category " + type
    else
      leaves.concat(some_leaves)
    end
  end 
  
  if leaves.empty?
    raise "No leaf nodes found for specified categories"
  end

  Rails.logger.info 'Found leaf nodes: ' + leaves.to_s

  start = Time.now
  # We first obtain the product list for each leaf category in series. Doing this step in parallel
  # seems to result in strange behavior for Best Buy's search API call.
  #
  # We store the product list for each leaf category in a temporary file and pass the path 
  # to the temp file to the child process responsible for scraping the category.
  #
  # We store temp file references in an array, to ensure they do not get garbage collected before 
  # child processes have a chance to read them (the temp files are deleted when they are garbage collected).
  temp_files = []
  curr_child_process_count = 0
  spawned_processes = 0
  leaves.each do |node|
    Session.new node
    products = Product.get_products(node)

    # Save product information to a temporary file
    temp_file = Tempfile.open("category_products", Rails.root.join('tmp')) { |temp_file|
      YAML.dump(products, temp_file)
      temp_file
    }
    temp_files << temp_file

    if curr_child_process_count >= 10 
      Process.wait
      curr_child_process_count -= 1
    end
    command_line = "bundle exec rake update_leaf product_type=#{node} file=#{temp_file.path}"
    if Rake.application.options.trace
      command_line += " --trace"
    end
    Process.spawn(command_line)
    curr_child_process_count += 1
    spawned_processes += 1
    Rails.logger.info "Spawned rake task for node " + node + " (" + spawned_processes.to_s + "/" + leaves.size.to_s + ")"
  end
                  
  Rails.logger.info "Finished spawning child processes"

  # Wait for all child processes to finish
  Process.waitall

  product_types.each do |product_type|
    Session.new product_type #Reset session
    #clean up inactive scraping rules not used any more
    Facet.check_active
  end

  Search.cleanup_history_data(7)

  #Report problem with script if it finishes too fast
  `touch tmp/r_updateproblem.txt` if (Time.now - start < 1.minute)

  puts "update_parallel finished at " + Time.now.to_s
end

# This task is not intended to be invoked directly. Instead it is invoked by 
# update_parallel for each leaf node under a category.
desc "Update data for a single leaf node"
task :update_leaf => :environment do
  
  unless ENV.include?("product_type") && ENV.include?("file")
    raise "usage: rake update_leaf product_type=? file=?"
  end
  
  node = ENV["product_type"]
  file_path = ENV["file"]

  # Force BBproduct class to be auto-loaded.
  BBproduct.new
  products = YAML.load_file(file_path)

  Session.new node
  begin 
    puts "update_leaf scraping " + products.size.to_s + " products for category " + node
    Product.feed_update(products)
    puts 'update_leaf finished scraping category ' + node
  rescue BestBuyApi::RequestError => error
    puts 'update_leaf got the following error in scraping category ' + node + ': ' + error.to_s
  end
end

namespace :cache do
  desc 'Clear memcache'
  task :clear => :environment do
{'fraction_non_alphanumeric': 0.08265871325095867, 'fraction_numerical': 0.00818065615679591, 'mean_word_length': 3.4370510396975424, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3671205', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3539, 'n_words': 1235}