<commit_before>namespace :grimes do
  desc "Scrape Heather Grimes' data."
  task :scrape => :environment do
    require 'scrape_utils'

    driver = Selenium::WebDriver.for :firefox
    driver.manage.timeouts.implicit_wait = 10
    customer = Customer.find_by_key("<PASSWORD>")

    url = "http://rentalhomepros.com/Membership.aspx"
    driver.get url

    xpath_un = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcLogin1_Login1_UserName\"]"
    driver.find_element(:xpath, xpath_un).send_keys "<EMAIL>"
    xpath_pw = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcLogin1_Login1_Password\"]"
    driver.find_element(:xpath, xpath_pw).send_keys "09031981"
    xpath_login = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcLogin1_Login1_LoginButton1\"]"
    driver.find_element(:xpath, xpath_login).click

    xpath_CL = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcSideNavBar1_SideBarNavTreen9\"]"
    driver.find_element(:xpath, xpath_CL).click

    active = []
    infos = {}
    total = driver.find_elements(:xpath, "/html/body/form/div[3]/div[2]/table/tbody/tr/td[2]/div[2]/div/table/tbody/tr[2]/td/div/table/tbody/tr[2]/td/table/tbody/tr/td[2]/select/option").count
    for i in (1..total)
      puts ",--"
      foreign_listing = driver.find_element(:xpath, "/html/body/form/div[3]/div[2]/table/tbody/tr/td[2]/div[2]/div/table/tbody/tr[2]/td/div/table/tbody/tr[2]/td/table/tbody/tr/td[2]/select/option[#{i}]")
      foreign_listing.click

      foreign_id = foreign_listing.attribute("value")
      puts "|Foreign ID: #{foreign_id}"
      listing = Listing.find_by_customer_id_and_foreign_id(customer.id, foreign_id)
      if listing.nil?
        listing = Listing.new
        listing.customer_id = customer.id
        listing.foreign_id = foreign_id
        listing.location_id = customer.location_id
        listing.sublocation_id = customer.sublocation_id
        listing.save
      end

      begin
        sleep(0.3)
      end until infos != get_ad_infos(driver)
      sleep(0.3)
      infos = get_ad_infos(driver)
      for key, val in infos
        value_update(listing, key, val)
      end
      listing.update_attribute(:foreign_active=>false) if listing.infos["ad_price"].to_i > 2500
      listing.save
      puts "|#{listing.errors}" if !listing.errors.nil? and listing.errors != {}
      active << listing.id
      puts "`--"
    end
    driver.quit
    activate_new_listings(customer.id, active)
    deactivate_old_listings(customer.id, active)
  end
<commit_msg>Grime's import now properly disables listings over 2500<commit_after>namespace :grimes do
  desc "Scrape Heather Grimes' data."
  task :scrape => :environment do
    require 'scrape_utils'

    driver = Selenium::WebDriver.for :firefox
    driver.manage.timeouts.implicit_wait = 10
    customer = Customer.find_by_key("heather_grimes")

    url = "http://rentalhomepros.com/Membership.aspx"
    driver.get url

    xpath_un = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcLogin1_Login1_UserName\"]"
    driver.find_element(:xpath, xpath_un).send_keys "<EMAIL>"
    xpath_pw = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcLogin1_Login1_Password\"]"
    driver.find_element(:xpath, xpath_pw).send_keys "09031981"
    xpath_login = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcLogin1_Login1_LoginButton1\"]"
    driver.find_element(:xpath, xpath_login).click

    xpath_CL = "//*[@id=\"ctl00_BodyContainerPlaceHolder_UcSideNavBar1_SideBarNavTreen9\"]"
    driver.find_element(:xpath, xpath_CL).click

    active = []
    infos = {}
    total = driver.find_elements(:xpath, "/html/body/form/div[3]/div[2]/table/tbody/tr/td[2]/div[2]/div/table/tbody/tr[2]/td/div/table/tbody/tr[2]/td/table/tbody/tr/td[2]/select/option").count
    for i in (1..total)
      puts ",--"
      foreign_listing = driver.find_element(:xpath, "/html/body/form/div[3]/div[2]/table/tbody/tr/td[2]/div[2]/div/table/tbody/tr[2]/td/div/table/tbody/tr[2]/td/table/tbody/tr/td[2]/select/option[#{i}]")
      foreign_listing.click

      foreign_id = foreign_listing.attribute("value")
      puts "|Foreign ID: #{foreign_id}"
      listing = Listing.find_by_customer_id_and_foreign_id(customer.id, foreign_id)
      if listing.nil?
        listing = Listing.new
        listing.customer_id = customer.id
        listing.foreign_id = foreign_id
        listing.location_id = customer.location_id
        listing.sublocation_id = customer.sublocation_id
        listing.save
      end

      begin
        sleep(0.3)
      end until infos != get_ad_infos(driver)
      sleep(0.3)
      infos = get_ad_infos(driver)
      for key, val in infos
        value_update(listing, key, val)
      end
      listing.save
      puts "|#{listing.errors}" if !listing.errors.nil? and listing.errors != {}
      active << listing.id if listing.infos["ad_price"].to_i <= 2500
      puts "`--"
    end
    driver.quit
    activate_new_listings(customer.id, active)
    deactivate_old_listings(customer.id, active)
  end

{'fraction_non_alphanumeric': 0.13862991487636805, 'fraction_numerical': 0.02107823267126064, 'mean_word_length': 3.462025316455696, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4293970', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1704, 'n_words': 244}