<commit_before>require 'csv'

namespace :property_data do
  desc "Import data from the property data file and create an addresses JSON file for typeahead search"
  task :import => :environment do
    csv_path = "/tmp/Vacant_and_Abandoned_Property_Data.csv"
    table = CSV.read(csv_path, :headers => true)
    all_address_array = []
    table.each do |row|
      parcel_id = row["Parcel ID"]
      puts "Processing #{parcel_id}"
      target_property = Property.find_by_parcel_id(parcel_id)
      address = row["Location 1"][0, row["Location 1"].index("\n")]
      all_address_array << address
      if target_property == nil
        target_property = Property.create(:parcel_id => parcel_id, :name => address)
      end
      recommendation = nil
      ["Repair","Demo","Deconstruct","Hold"].each do |key|
        if (recommendation && row[key])
          puts "Warning! #{row["Parcel ID"]} has multiple recommendations"
          binding.pry
        end
        recommendation = key if row[key] == "1"
      end
      outcome = nil
      ["Repaired","Demolished","Deconstructed","Occupied / Repaired","Occupied / Not Repaired","Legal Hold"].each do |key|
        if (outcome && row[key])
          puts "Warning! #{row["Parcel ID"]} has multiple outcomes"
          binding.pry
        end
        outcome = key if row[key] == "1"
      end
      outcome = "Vacant and Abandoned" if outcome == nil
      if target_property.property_info_set
        target_property.property_info_set.update_attributes(:condition_code => row["Condition Code"].to_i, :condition => row["Condition (auto populates)"], :estimated_cost_exterior=> row["Estimated cost (Exterior)"], :estimated_cost_interior => row["Estimated cost (Interior - if able)"], :demo_order => row["Demo order? (Affirmed/Expired)"], :recommendation => recommendation, :outcome => outcome)
      else # Already has property info set
        target_property.property_info_set = PropertyInfoSet.create(:condition_code => row["Condition Code"].to_i, :condition => row["Condition (auto populates)"], :estimated_cost_exterior=> row["Estimated cost (Exterior)"], :estimated_cost_interior => row["Estimated cost (Interior - if able)"], :demo_order => row["Demo order? (Affirmed/Expired)"], :recommendation => recommendation, :outcome => outcome)
      end
    end
    address_json_path = "#{Rails.root}/app/assets/javascripts/property_addresses.json"
    File.delete(address_json_path) if File.exist?(address_json_path)
    File.open(address_json_path, 'w') { |file| file.write(all_address_array.to_json) }
  end
  desc "PENDING - Pull down CSV data from Socrata and store in /tmp"
  task :download_from_socrata do
    # Pending
  end
end

<commit_msg>Rake task now adds lat and long for properties<commit_after>require 'csv'

namespace :property_data do
  desc "Import data from the property data file and create an addresses JSON file for typeahead search"
  task :import => :environment do
    csv_path = "/tmp/Vacant_and_Abandoned_Property_Data.csv"
    table = CSV.read(csv_path, :headers => true)
    all_address_array = []
    table.each do |row|
      parcel_id = row["Parcel ID"]
      puts "Processing #{parcel_id}"
      target_property = Property.find_by_parcel_id(parcel_id)
      address = row["Location 1"][0, row["Location 1"].index("\n")]
      all_address_array << address
      if target_property == nil
        target_property = Property.create(:parcel_id => parcel_id, :name => address)
      end
      latlong = row["Location 1"][/\((.*)\)/]
      lat = latlong[/\((.*)[,]/].gsub(/([\(]|[,])/, "")
      long = latlong[/\s(.*)$/].gsub(/(\s|\))/, "")
      recommendation = nil
      ["Repair","Demo","Deconstruct","Hold"].each do |key|
        if (recommendation && row[key])
          puts "Warning! #{row["Parcel ID"]} has multiple recommendations"
          binding.pry
        end
        recommendation = key if row[key] == "1"
      end
      outcome = nil
      ["Repaired","Demolished","Deconstructed","Occupied / Repaired","Occupied / Not Repaired","Legal Hold"].each do |key|
        if (outcome && row[key])
          puts "Warning! #{row["Parcel ID"]} has multiple outcomes"
          binding.pry
        end
        outcome = key if row[key] == "1"
      end
      outcome = "Vacant and Abandoned" if outcome == nil
      if target_property.property_info_set
        target_property.property_info_set.update_attributes(:condition_code => row["Condition Code"].to_i, :condition => row["Condition (auto populates)"], :estimated_cost_exterior=> row["Estimated cost (Exterior)"], :estimated_cost_interior => row["Estimated cost (Interior - if able)"], :demo_order => row["Demo order? (Affirmed/Expired)"], :recommendation => recommendation, :outcome => outcome, :lat => lat, :long => long)
      else # Already has property info set
        target_property.property_info_set = PropertyInfoSet.create(:condition_code => row["Condition Code"].to_i, :condition => row["Condition (auto populates)"], :estimated_cost_exterior=> row["Estimated cost (Exterior)"], :estimated_cost_interior => row["Estimated cost (Interior - if able)"], :demo_order => row["Demo order? (Affirmed/Expired)"], :recommendation => recommendation, :outcome => outcome, :lat => lat, :long => long)
      end
    end
    address_json_path = "#{Rails.root}/app/assets/javascripts/property_addresses.json"
    File.delete(address_json_path) if File.exist?(address_json_path)
    File.open(address_json_path, 'w') { |file| file.write(all_address_array.to_json) }
  end
  desc "PENDING - Pull down CSV data from Socrata and store in /tmp"
  task :download_from_socrata do
    # Pending
  end
end


{'fraction_non_alphanumeric': 0.1373070782331027, 'fraction_numerical': 0.001951392584708178, 'mean_word_length': 3.7578059071729957, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5081759', 'n_tokens_mistral': 1740, 'n_tokens_neox': 1723, 'n_words': 435}