<commit_before>class GeocoderController < ApplicationController
  require 'uri'
  require 'net/http'
  require 'rexml/document'

  POSTCODE_ZOOM = 15
  GEONAMES_ZOOM = 12

  def search
    query = params[:query]
    results = Array.new

    if query.match(/^\d{5}(-\d{4})?$/)
      results.push search_us_postcode(query)
    elsif query.match(/(GIR 0AA|[A-PR-UWYZ]([0-9]{1,2}|([A-HK-Y][0-9]|[A-HK-Y][0-9]([0-9]|[ABEHMNPRV-Y]))|[0-9][A-HJKS-UW])\s*[0-9][ABD-HJLNP-UW-Z]{2})/i)
      results.push search_uk_postcode(query)
      results.push search_osm_namefinder(query)
    elsif query.match(/[A-Z]\d[A-Z]\s*\d[A-Z]\d/i)
      results.push search_ca_postcode(query)
    else
      results.push search_osm_namefinder(query)
      results.push search_geonames(query)
    end

    results_count = count_results(results)

    render :update do |page|
      page.replace_html :sidebar_content, :partial => 'results', :object => results

      if results_count == 1
        position = results.collect { |s| s[:results] }.compact.flatten[0]
        page.call "setPosition", position[:lat], position[:lon], position[:zoom]
      else
        page.call "openSidebar"
      end
    end
  end
  
  def description
    results = Array.new

    lat = params[:lat]
    lon = params[:lon]

    results.push description_osm_namefinder("cities", lat, lon, 2)
    results.push description_osm_namefinder("towns", lat, lon, 4)
    results.push description_osm_namefinder("places", lat, lon, 10)
    results.push description_geonames(lat, lon)

    render :update do |page|
      page.replace_html :sidebar_content, :partial => 'results', :object => results
      page.call "openSidebar"
    end
  end

private

  def search_us_postcode(query)
    results = Array.new

    # ask geocoder.us (they have a non-commercial use api)
    response = fetch_text("http://rpc.geocoder.us/service/csv?zip=#{escape_query(query)}")

    # parse the response
    unless response.match(/couldn't find this zip/)
      data = response.split(/\s*,\s+/) # lat,long,town,state,zip
      results.push({:lat => data[0], :lon => data[1], :zoom => POSTCODE_ZOOM,
                    :prefix => "#{data[2]}, #{data[3]}, ",
                    :name => data[4]})
    end

    return { :source => "Geocoder.us", :url => "http://geocoder.us/", :results => results }
  rescue Exception => ex
    return { :source => "Geocoder.us", :url => "http://geocoder.us/", :error => "Error contacting rpc.geocoder.us: #{ex.to_s}" }
  end

  def search_uk_postcode(query)
    results = Array.new

    # ask npemap.org.uk to do a combined npemap + freethepostcode search
    response = fetch_text("http://www.npemap.org.uk/cgi/geocoder.fcgi?format=text&postcode=#{escape_query(query)}")

    # parse the response
    unless response.match(/Error/)
      dataline = response.split(/\n/)[1]
      data = dataline.split(/,/) # easting,northing,postcode,lat,long
      postcode = data[2].gsub(/'/, "")
      zoom = POSTCODE_ZOOM - postcode.count("#")
      results.push({:lat => data[3], :lon => data[4], :zoom => zoom,
                    :name => postcode})
    end

    return { :source => "NPEMap / FreeThePostcode", :url => "http://www.npemap.org.uk/", :results => results }
  rescue Exception => ex
    return { :source => "NPEMap / FreeThePostcode", :url => "http://www.npemap.org.uk/", :error => "Error contacting www.npemap.org.uk: #{ex.to_s}" }
  end

  def search_ca_postcode(query)
    results = Array.new

    # ask geocoder.ca (note - they have a per-day limit)
    response = fetch_xml("http://geocoder.ca/?geoit=XML&postal=#{escape_query(query)}")

    # parse the response
    unless response.get_elements("geodata/error")
      results.push({:lat => response.get_text("geodata/latt").to_s,
                    :lon => response.get_text("geodata/longt").to_s,
                    :zoom => POSTCODE_ZOOM,
                    :name => query.upcase})
    end

    return { :source => "Geocoder.CA", :url => "http://geocoder.ca/", :results => results }
  rescue Exception => ex
    return { :source => "Geocoder.CA", :url => "http://geocoder.ca/", :error => "Error contacting geocoder.ca: #{ex.to_s}" }
  end

  def search_osm_namefinder(query)
    results = Array.new

    # ask OSM namefinder
    response = fetch_xml("http://gazetteer.openstreetmap.org/namefinder/search.xml?find=#{escape_query(query)}")

    # parse the response
    response.elements.each("searchresults/named") do |named|
      lat = named.attributes["lat"].to_s
      lon = named.attributes["lon"].to_s
      zoom = named.attributes["zoom"].to_s
      place = named.elements["place/named"] || named.elements["nearestplaces/named"]
      type = named.attributes["info"].to_s.capitalize
      name = named.attributes["name"].to_s
      description = named.elements["description"].to_s

      if name.empty?
        prefix = ""
        name = type
      else
        prefix = "#{type} "
      end

      if place
        distance = format_distance(place.attributes["approxdistance"].to_i)
        direction = format_direction(place.attributes["direction"].to_i)
        placename = format_name(place.attributes["name"].to_s)
        suffix = ", #{distance} #{direction} of #{placename}"

        if place.attributes["rank"].to_i <= 30
          parent = nil
          parentrank = 0
          parentdistance = 0

          place.elements.each("nearestplaces/named") do |nearest|
            nearestrank = nearest.attributes["rank"].to_i
            nearestdistance = nearest.attributes["distance"].to_f

            if nearestrank > parentrank or
               ( nearestrank == parentrank and nearestdistance < parentdistance )
              parent = nearest
              parentrank = nearestrank
              parentdistance = nearestdistance
            end
          end

          if parent
            parentname = format_name(parent.attributes["name"].to_s)

            if  place.attributes["info"].to_s == "suburb"
              suffix = "#{suffix}, #{parentname}"
            else
              suffix = "#{suffix} (near #{parentname})"
            end
          end
        end
      else
        suffix = ""
      end

      results.push({:lat => lat, :lon => lon, :zoom => zoom,
                    :prefix => prefix, :name => name, :suffix => suffix,
                    :description => description})
    end

    return { :source => "OpenStreetMap Namefinder", :url => "http://gazetteer.openstreetmap.org/namefinder/", :results => results }
  rescue Exception => ex
    return { :source => "OpenStreetMap Namefinder", :url => "http://gazetteer.openstreetmap.org/namefinder/", :error => "Error contacting gazetteer.openstreetmap.org: #{ex.to_s}" }
  end

  def search_geonames(query)
    results = Array.new

    # ask geonames.org
    response = fetch_xml("http://ws.geonames.org/search?q=#{escape_query(query)}&maxRows=20")

    # parse the response
    response.elements.each("geonames/geoname") do |geoname|
      lat = geoname.get_text("lat").to_s
      lon = geoname.get_text("lng").to_s
      name = geoname.get_text("name").to_s<commit_msg>Tweak the namefinder results a bit more.<commit_after>class GeocoderController < ApplicationController
  require 'uri'
  require 'net/http'
  require 'rexml/document'

  POSTCODE_ZOOM = 15
  GEONAMES_ZOOM = 12

  def search
    query = params[:query]
    results = Array.new

    if query.match(/^\d{5}(-\d{4})?$/)
      results.push search_us_postcode(query)
    elsif query.match(/(GIR 0AA|[A-PR-UWYZ]([0-9]{1,2}|([A-HK-Y][0-9]|[A-HK-Y][0-9]([0-9]|[ABEHMNPRV-Y]))|[0-9][A-HJKS-UW])\s*[0-9][ABD-HJLNP-UW-Z]{2})/i)
      results.push search_uk_postcode(query)
      results.push search_osm_namefinder(query)
    elsif query.match(/[A-Z]\d[A-Z]\s*\d[A-Z]\d/i)
      results.push search_ca_postcode(query)
    else
      results.push search_osm_namefinder(query)
      results.push search_geonames(query)
    end

    results_count = count_results(results)

    render :update do |page|
      page.replace_html :sidebar_content, :partial => 'results', :object => results

      if results_count == 1
        position = results.collect { |s| s[:results] }.compact.flatten[0]
        page.call "setPosition", position[:lat], position[:lon], position[:zoom]
      else
        page.call "openSidebar"
      end
    end
  end
  
  def description
    results = Array.new

    lat = params[:lat]
    lon = params[:lon]

    results.push description_osm_namefinder("cities", lat, lon, 2)
    results.push description_osm_namefinder("towns", lat, lon, 4)
    results.push description_osm_namefinder("places", lat, lon, 10)
    results.push description_geonames(lat, lon)

    render :update do |page|
      page.replace_html :sidebar_content, :partial => 'results', :object => results
      page.call "openSidebar"
    end
  end

private

  def search_us_postcode(query)
    results = Array.new

    # ask geocoder.us (they have a non-commercial use api)
    response = fetch_text("http://rpc.geocoder.us/service/csv?zip=#{escape_query(query)}")

    # parse the response
    unless response.match(/couldn't find this zip/)
      data = response.split(/\s*,\s+/) # lat,long,town,state,zip
      results.push({:lat => data[0], :lon => data[1], :zoom => POSTCODE_ZOOM,
                    :prefix => "#{data[2]}, #{data[3]}, ",
                    :name => data[4]})
    end

    return { :source => "Geocoder.us", :url => "http://geocoder.us/", :results => results }
  rescue Exception => ex
    return { :source => "Geocoder.us", :url => "http://geocoder.us/", :error => "Error contacting rpc.geocoder.us: #{ex.to_s}" }
  end

  def search_uk_postcode(query)
    results = Array.new

    # ask npemap.org.uk to do a combined npemap + freethepostcode search
    response = fetch_text("http://www.npemap.org.uk/cgi/geocoder.fcgi?format=text&postcode=#{escape_query(query)}")

    # parse the response
    unless response.match(/Error/)
      dataline = response.split(/\n/)[1]
      data = dataline.split(/,/) # easting,northing,postcode,lat,long
      postcode = data[2].gsub(/'/, "")
      zoom = POSTCODE_ZOOM - postcode.count("#")
      results.push({:lat => data[3], :lon => data[4], :zoom => zoom,
                    :name => postcode})
    end

    return { :source => "NPEMap / FreeThePostcode", :url => "http://www.npemap.org.uk/", :results => results }
  rescue Exception => ex
    return { :source => "NPEMap / FreeThePostcode", :url => "http://www.npemap.org.uk/", :error => "Error contacting www.npemap.org.uk: #{ex.to_s}" }
  end

  def search_ca_postcode(query)
    results = Array.new

    # ask geocoder.ca (note - they have a per-day limit)
    response = fetch_xml("http://geocoder.ca/?geoit=XML&postal=#{escape_query(query)}")

    # parse the response
    unless response.get_elements("geodata/error")
      results.push({:lat => response.get_text("geodata/latt").to_s,
                    :lon => response.get_text("geodata/longt").to_s,
                    :zoom => POSTCODE_ZOOM,
                    :name => query.upcase})
    end

    return { :source => "Geocoder.CA", :url => "http://geocoder.ca/", :results => results }
  rescue Exception => ex
    return { :source => "Geocoder.CA", :url => "http://geocoder.ca/", :error => "Error contacting geocoder.ca: #{ex.to_s}" }
  end

  def search_osm_namefinder(query)
    results = Array.new

    # ask OSM namefinder
    response = fetch_xml("http://gazetteer.openstreetmap.org/namefinder/search.xml?find=#{escape_query(query)}")

    # parse the response
    response.elements.each("searchresults/named") do |named|
      lat = named.attributes["lat"].to_s
      lon = named.attributes["lon"].to_s
      zoom = named.attributes["zoom"].to_s
      place = named.elements["place/named"] || named.elements["nearestplaces/named"]
      type = named.attributes["info"].to_s.capitalize
      name = named.attributes["name"].to_s
      description = named.elements["description"].to_s

      if name.empty?
        prefix = ""
        name = type
      else
        prefix = "#{type} "
      end

      if place
        distance = format_distance(place.attributes["approxdistance"].to_i)
        direction = format_direction(place.attributes["direction"].to_i)
        placename = format_name(place.attributes["name"].to_s)
        suffix = ", #{distance} #{direction} of #{placename}"

        if place.attributes["rank"].to_i <= 30
          parent = nil
          parentrank = 0
          parentscore = 0

          place.elements.each("nearestplaces/named") do |nearest|
            nearestrank = nearest.attributes["rank"].to_i
            nearestscore = nearestrank / nearest.attributes["distance"].to_f

            if nearestrank > 30 and
               ( nearestscore > parentscore or
                 ( nearestscore == parentscore and nearestrank > parentrank ) )
              parent = nearest
              parentrank = nearestrank
              parentscore = nearestscore
            end
          end

          if parent
            parentname = format_name(parent.attributes["name"].to_s)

            if  place.attributes["info"].to_s == "suburb"
              suffix = "#{suffix}, #{parentname}"
            else
              parentdistance = format_distance(parent.attributes["approxdistance"].to_i)
              parentdirection = format_direction(parent.attributes["direction"].to_i)
              suffix = "#{suffix} (#{parentdistance} #{parentdirection} of #{parentname})"
            end
          end
        end
      else
        suffix = ""
      end

      results.push({:lat => lat, :lon => lon, :zoom => zoom,
                    :prefix => prefix, :name => name, :suffix => suffix,
                    :description => description})
    end

    return { :source => "OpenStreetMap Namefinder", :url => "http://gazetteer.openstreetmap.org/namefinder/", :results => results }
  rescue Exception => ex
    return { :source => "OpenStreetMap Namefinder", :url => "http://gazetteer.openstreetmap.org/namefinder/", :error => "Error contacting gazetteer.openstreetmap.org: #{ex.to_s}" }
  end

  def search_geonames(query)
    results = Array.new

    # ask geonames.org
    response = fetch_xml("http://ws.geonames.org/search?q=#{escape_query(query)}&maxRows=20")

    # parse the response
    response.elements.each("geonames/geoname") do |geoname|
      lat = geoname.get_text("lat").to_s
      lon = geoname.get_text("lng").to_s
      name = geoname.get_text("name").to_s
{'fraction_non_alphanumeric': 0.15021815915229586, 'fraction_numerical': 0.006094604889535287, 'mean_word_length': 2.7448132780082988, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '395498', 'n_tokens_mistral': 4814, 'n_tokens_neox': 4597, 'n_words': 905}