<commit_before>def search_for(search_type, params)
# Accesses the Amazon feed and returns 10 products and their data
# The data is split into an array and returned
  associates_id = '***REMOVED***'
  key_id = '***REMOVED***'
  is = ItemSearch.new(search_type, params)
  is.response_group = ResponseGroup.new( 'Medium', 'Images', 'ItemAttributes')
  resp = Request.new(key_id, associates_id).search( is )
  string_array = resp.item_search_response[0].to_s.gsub!("\n", '|').split('|')
end

def create_amazon_specs(item, id)
# Creates all specs for a given item
  # Create product_type (not a rule in scraping rules)
  c = CatSpec.find_or_initialize_by_product_id_and_name(id, 'product_type')
  c.update_attributes(value: item['product_type'])
  c = ContSpec.find_or_initialize_by_product_id_and_name(id, 'saleprice')
  c.update_attributes(value: item['saleprice'])
  # Get all the rules that apply to the parent category or this item's category
  for rule in ScrapingRule.find_by_sql("select * from `scraping_rules` where `product_type` in ('ADepartments', '#{item['product_type']}')")
    rule_type = CatSpec
    case rule.rule_type
    when 'Continuous'
      rule_type = ContSpec
    when 'Binary'
      rule_type = BinSpec
    when 'Text'
      rule_type = TextSpec
    end
    if item[rule.local_featurename]
      spec = rule_type.find_or_initialize_by_product_id_and_name(id, rule.local_featurename)
      spec.update_attributes(value: item[rule.local_featurename])\
    end
  end
end

def get_scraping_rules(product_type)
# Gathers all scraping rules for a given product type as well as for the parent department (in this case ADepartments)
  scraping_rules = []
  scraping_rules << ScrapingRule.where(product_type: product_type)
  scraping_rules << ScrapingRule.where(product_type: 'ADepartments')
  scraping_rules.flatten!
end

def get_real_rule(scraping_rule)
# Parses the scraping rule and extracts the regex and any additional information
  scraping_rule.gsub!(/^\//, "")
  index = scraping_rule.index(/[^\\]\//)
  if index # If the regex contains an unescaped /
    # Then extract the regex and whatever information comes after the /
    regexp = Regexp.new(scraping_rule[0..index])
    replacement = scraping_rule[index+2...scraping_rule.length]
    gsub = false
    gsub = true if replacement =~ /^\\/ # Indicates a rule that looks like this: '/(.*)(regex)(.*)/\2', one that will have partial data extracted from it later
                                        # If gsub is false it means that the rule is something more like '/regex/REPLACEMENT'
                                        # => where if the text matches the regex, the extracted data will simply be 'REPLACEMENT'
    rule = [regexp, replacement, gsub]
  else
    rule = [Regexp.new(scraping_rule)]
  end
end

def save(items)
# Uploads data for all items
  retailer = "A"
  total_items = items.length
  completed = 0
  products_to_save = []
  for item in items
    if item['title'] # Don't upload products without names
      puts "Uploading to database: #{completed*100/total_items}%"
      # Make sure not to duplicate products - check if they're already in the table
      prod = Product.find_or_initialize_by_sku_and_retailer(item['sku'], retailer)
      prod.update_attributes(instock: true)
      
      create_amazon_specs(item, Product.find_by_sku_and_retailer(item['sku'], retailer).id)
      products_to_save << prod
      completed += 1
    end
  end
  
  puts 'running custom rules and equivalences'
  # save equivalences
  categories = ['Acamera_amazon', 'Atv_amazon', 'Amovie_amazon', 'Asoftware_amazon']
  
  categories.each do |category|
    Session.new category
    custom_specs_to_save = Customization.run(products_to_save.map(&:id))
    custom_specs_to_save.each do |spec_class, spec_values|
      spec_class.import spec_values, :on_duplicate_key_update=>[:product_id, :name, :value]
    end
    Equivalence.fill
  end
  
  puts 'indexing products to sunspot'
    
  Sunspot.index(products_to_save)
  Sunspot.commit
  puts "Sunspot now has #{Sunspot.search(Product).total} def products"
  puts "Save complete: #{Product.where(retailer: retailer).length} products saved"
end

def scrape(product_type, string_array)
# Uses the scraping rules to gather all data from the Amazon string array
  scraping_rules = get_scraping_rules(product_type)
  i = 0
  item_index = -1
  items = []
  while i < string_array.length
    text = string_array[i].gsub('&amp;', '&')
    
    if text =~ /asin = / || i == string_array.length-1 # If we've parsed a full item
      if item_index >= 0 # If at least one item has been parsed (we don't want to accidentally try to go to items[-1])
        # Make adjustments to the previous item
        
        # Overwrite the product type: need a scraping rule for the layout, but the department (product_group) we get from amazon
        # is not enough to differentiate between our product types
        items[item_index]['product_type'] = product_type
        # Fix formatting
        items[item_index]['screen_type'].upcase! if items[item_index]['screen_type'] && items[item_index]['screen_type'] !~ /plasma/i
        items[item_index]['price'] = items[item_index]['price'].to_i/100.0 if items[item_index]['price']
        items[item_index]['price_new'] = items[item_index]['price_new'].to_i/100.0 if items[item_index]['price_new']<commit_msg>Adjusted Amazon scraping and utility rule for Amazon.<commit_after>def search_for(search_type, params)
# Accesses the Amazon feed and returns 10 products and their data
# The data is split into an array and returned
  associates_id = '***REMOVED***'
  key_id = '***REMOVED***'
  is = ItemSearch.new(search_type, params)
  is.response_group = ResponseGroup.new( 'Medium', 'Images', 'ItemAttributes')
  resp = Request.new(key_id, associates_id).search( is )
  string_array = resp.item_search_response[0].to_s.gsub!("\n", '|').split('|')
end

def create_amazon_specs(item, id)
# Creates all specs for a given item
  # Create product_type (not a rule in scraping rules)
  c = CatSpec.find_or_initialize_by_product_id_and_name(id, 'product_type')
  c.update_attributes(value: item['product_type'])
  c = ContSpec.find_or_initialize_by_product_id_and_name(id, 'saleprice')
  c.update_attributes(value: item['saleprice'])
  # Get all the rules that apply to the parent category or this item's category
  for rule in ScrapingRule.find_by_sql("select * from `scraping_rules` where `product_type` in ('ADepartments', '#{item['product_type']}')")
    rule_type = CatSpec
    case rule.rule_type
    when 'Continuous'
      rule_type = ContSpec
    when 'Binary'
      rule_type = BinSpec
    when 'Text'
      rule_type = TextSpec
    end
    if item[rule.local_featurename]
      spec = rule_type.find_or_initialize_by_product_id_and_name(id, rule.local_featurename)
      spec.update_attributes(value: item[rule.local_featurename])\
    end
  end
end

def get_scraping_rules(product_type)
# Gathers all scraping rules for a given product type as well as for the parent department (in this case ADepartments)
  scraping_rules = []
  scraping_rules << ScrapingRule.where(product_type: product_type)
  scraping_rules << ScrapingRule.where(product_type: 'ADepartments')
  scraping_rules.flatten!
end

def get_real_rule(scraping_rule)
# Parses the scraping rule and extracts the regex and any additional information
  scraping_rule.gsub!(/^\//, "")
  index = scraping_rule.index(/[^\\]\//)
  if index # If the regex contains an unescaped /
    # Then extract the regex and whatever information comes after the /
    regexp = Regexp.new(scraping_rule[0..index])
    replacement = scraping_rule[index+2...scraping_rule.length]
    gsub = false
    gsub = true if replacement =~ /^\\/ # Indicates a rule that looks like this: '/(.*)(regex)(.*)/\2', one that will have partial data extracted from it later
                                        # If gsub is false it means that the rule is something more like '/regex/REPLACEMENT'
                                        # => where if the text matches the regex, the extracted data will simply be 'REPLACEMENT'
    rule = [regexp, replacement, gsub]
  else
    rule = [Regexp.new(scraping_rule)]
  end
end

def save(items)
# Uploads data for all items
  retailer = "A"
  total_items = items.length
  completed = 0
  products_to_save = []
  for item in items
    if item['title'] # Don't upload products without names
      puts "Uploading to database: #{completed*100/total_items}%"
      # Make sure not to duplicate products - check if they're already in the table
      prod = Product.find_or_initialize_by_sku_and_retailer(item['sku'], retailer)
      prod.update_attributes(instock: true)
      
      create_amazon_specs(item, Product.find_by_sku_and_retailer(item['sku'], retailer).id)
      products_to_save << prod
      completed += 1
    end
  end
  
  puts 'running custom rules and equivalences'
  # save equivalences
  categories = ['Acamera_amazon', 'Atv_amazon', 'Amovie_amazon', 'Asoftware_amazon']
  
  categories.each do |category|
    Session.new category
    custom_specs_to_save = Customization.run(products_to_save.map(&:id))
    custom_specs_to_save.each do |spec_class, spec_values|
      spec_class.import spec_values, :on_duplicate_key_update=>[:product_id, :name, :value]
    end
    Equivalence.fill
  end
  
  puts 'indexing products to sunspot'
    
  Sunspot.index(products_to_save)
  Sunspot.commit
  puts "Sunspot now has #{Sunspot.search(Product).total} def products"
  puts "Save complete: #{Product.where(retailer: retailer).length} products saved"
end

def scrape(product_type, string_array)
# Uses the scraping rules to gather all data from the Amazon string array
  scraping_rules = get_scraping_rules(product_type)
  i = 0
  item_index = -1
  items = []
  while i < string_array.length
    text = string_array[i].gsub('&amp;', '&')
    
    if text =~ /asin = / || i == string_array.length-1 # If we've parsed a full item
      if item_index >= 0 # If at least one item has been parsed (we don't want to accidentally try to go to items[-1])
        # Make adjustments to the previous item
        
        # Overwrite the product type: need a scraping rule for the layout, but the department (product_group) we get from amazon
        # is not enough to differentiate between our product types
        items[item_index]['product_type'] = product_type
        # Fix formatting
        items[item_index]['screen_type'].upcase! if items[item_index]['screen_type'] && items[item_index]['screen_type'] !~ /plasma/i
        items[item_index]['hd_video'].downcase! if items[item_index]['hd_video']
        items[item_index]['resolution'].downcase! if items[item_index]['resolution']
        items[item_index]['price'] = items[item_index]['price'].to_i/100.0 if items[item_index]['price']
        items[item_index]['price_new'] = items[item_index]['price_new'].to_i/100.0 if items[item_index]['price_new']
{'fraction_non_alphanumeric': 0.10396854425749817, 'fraction_numerical': 0.0043891733723482075, 'mean_word_length': 3.867378727191811, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3489077', 'n_tokens_mistral': 3472, 'n_tokens_neox': 3420, 'n_words': 964}