<commit_before>require 'open-uri'
require 'mechanize'
require 'listing_title'
require 'scrape'

class Mlxchange < Scrape
  COORDINATES_MAP = {
    "168,56" => { # 8 Photo View
      "120,192" => "ad_address",
      "824,16" => "ad_amenities",
      "1000,16" => "ad_attribution",
      "288,120" => "ad_bedrooms",
      "552,16" => "ad_description",
      "760,16" => "ad_exterior",
      "304,120" => "ad_full_bathrooms",
      "336,120" => "ad_half_bathrooms",
      "696,16" => "ad_interior",
      "120,24" => "ad_location",
      "120,608" => "ad_price",
      "352,120" => "ad_square_feet",
      "192,72" => "ad_status",
      "256,16" => "ad_subdivision",
      "504,104" => "ad_waterfront",
    },
    "176,64" => {
      "16,200" => "ad_address",
      "792,520" => "ad_amenities",
      "992,16" => "ad_attribution",
      "200,104" => "ad_bedrooms",
      "176,520" => "ad_complex",
      "280,16" => "ad_description",
      "648,520" => "ad_exterior",
      "528,120" => "ad_equipment",
      "224,10" => "ad_full_bathrooms",
      "248,104" => "ad_half_bathrooms",
      "480,120" => "ad_interior",
      "16,16" => "ad_location",
      "632,520" => "ad_pets",
      "16,616" => "ad_price",
      "176,328" => "ad_square_feet",
      "48,240" => "ad_status",
      "208,328" => "ad_type",
      "728,144" => "ad_waterfont",
      "64,496" => "ad_zip",
    },
    "56,391" =>
    {
      "56,89" => "ad_address",
      "792,78" => "ad_amenities",
      "444,138" => "ad_attribution",
      "282,114" => "ad_bedrooms",
      "534,12" => "ad_description",
      "696,78" => "ad_equipment",
      "720,78" => "ad_exterior",
      "294,114" => "ad_full_bathrooms",
      "306,114" => "ad_half_bathrooms",
      "672,78" => "ad_interior",
      "114,300" => "ad_location",
      "204,450" => "ad_pets",
      "81,634" => "ad_price",
      "330,114" => "ad_square_feet",
      "92,625" => "ad_status",
      "204,354" => "ad_waterfront",
      "114,636" => "ad_zip",
    }
  }
  #info is a hash in the form:
  # :data => [{:url=>"scrape_url",:infos=>{}},] # One inner hash per url
  # :customer_key => 'customer_key'
  # :disable_new_titles => bool (generate new titles (true/false)?)
  # :activate_new => bool (activate newly imported listings)
  # :deactivate_old => bool (deactivate old listings)
  def mlx_import
    agent = Mechanize.new

    disable_new_titles = @import_run.input_parsed["disable_new_titles"]
    customer_key = @import_run.input_parsed["customer_key"]
    activate_new = @import_run.input_parsed["activate_new"]
    deactivate_old = @import_run.input_parsed["deactivate_old"]
    begin
      customer_id = Customer.where("key like ?",customer_key).first.id 
    rescue 
      @output["failed"] = "Customer Key '#{customer_key.to_s}' Not Found"
      return false
    end

    active = []
    $page = nil<commit_msg>Add support for real estate 8-photo view<commit_after>require 'open-uri'
require 'mechanize'
require 'listing_title'
require 'scrape'

class Mlxchange < Scrape
  COORDINATES_MAP = {
    "168,56" => { # 8 Photo View
      "120,192" => "ad_address",
      "824,16" => "ad_amenities",
      "1000,16" => "ad_attribution",
      "288,120" => "ad_bedrooms",
      "552,16" => "ad_description",
      "760,16" => "ad_exterior",
      "304,120" => "ad_full_bathrooms",
      "336,120" => "ad_half_bathrooms",
      "696,16" => "ad_interior",
      "120,24" => "ad_location",
      "120,608" => "ad_price",
      "352,120" => "ad_square_feet",
      "192,72" => "ad_status",
      "256,16" => "ad_subdivision",
      "504,104" => "ad_waterfront",
    },
    "176,64" => {
      "16,200" => "ad_address",
      "792,520" => "ad_amenities",
      "992,16" => "ad_attribution",
      "200,104" => "ad_bedrooms",
      "176,520" => "ad_complex",
      "280,16" => "ad_description",
      "648,520" => "ad_exterior",
      "528,120" => "ad_equipment",
      "224,10" => "ad_full_bathrooms",
      "248,104" => "ad_half_bathrooms",
      "480,120" => "ad_interior",
      "16,16" => "ad_location",
      "632,520" => "ad_pets",
      "16,616" => "ad_price",
      "176,328" => "ad_square_feet",
      "48,240" => "ad_status",
      "208,328" => "ad_type",
      "728,144" => "ad_waterfont",
      "64,496" => "ad_zip",
    },
    "56,391" =>
    {
      "56,89" => "ad_address",
      "792,78" => "ad_amenities",
      "444,138" => "ad_attribution",
      "282,114" => "ad_bedrooms",
      "534,12" => "ad_description",
      "696,78" => "ad_equipment",
      "720,78" => "ad_exterior",
      "294,114" => "ad_full_bathrooms",
      "306,114" => "ad_half_bathrooms",
      "672,78" => "ad_interior",
      "114,300" => "ad_location",
      "204,450" => "ad_pets",
      "81,634" => "ad_price",
      "330,114" => "ad_square_feet",
      "92,625" => "ad_status",
      "204,354" => "ad_waterfront",
      "114,636" => "ad_zip",
    },
    "160,64" =>
    {
      "120,192" => "ad_address",
      "752,24" => "ad_amenities",
      "992,24" => "ad_attribution",
      "248,128" => "ad_bedrooms",
      "224,24" => "ad_complex",
      "480,120" => "ad_cooling",
      "504,24" => "ad_description",
      "272,128" => "ad_full_bathrooms",
      "296,128" => "ad_half_bathrooms",
      "688,24" => "ad_interior",
      "120,24" => "ad_location",
      "896,24" => "ad_maintenance",
      "120,600" => "ad_price",
      "320,128" => "ad_square_feet",
      "176,80" => "ad_status",
      "456,120" => "ad_waterfront"
    }
  }
  #info is a hash in the form:
  # :data => [{:url=>"scrape_url",:infos=>{}},] # One inner hash per url
  # :customer_key => 'customer_key'
  # :disable_new_titles => bool (generate new titles (true/false)?)
  # :activate_new => bool (activate newly imported listings)
  # :deactivate_old => bool (deactivate old listings)
  def mlx_import
    agent = Mechanize.new

    disable_new_titles = @import_run.input_parsed["disable_new_titles"]
    customer_key = @import_run.input_parsed["customer_key"]
    activate_new = @import_run.input_parsed["activate_new"]
    deactivate_old = @import_run.input_parsed["deactivate_old"]
    begin
      customer_id = Customer.where("key like ?",customer_key).first.id 
    rescue 
      @output["failed"] = "Customer Key '#{customer_key.to_s}' Not Found"
      return false
    end

    active = []
    $page = nil
{'fraction_non_alphanumeric': 0.19866708981275785, 'fraction_numerical': 0.10917169152649953, 'mean_word_length': 3.0507712082262213, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 142, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2723640', 'n_tokens_mistral': 2907, 'n_tokens_neox': 2435, 'n_words': 404}