<commit_before>#!/usr/bin/perl
#
# FixMyStreet:Geocode::OSM
# OpenStreetmap forward and reverse geocoding for FixMyStreet.
#
# Copyright (c) 2011 <NAME>. Some rights reserved.
# Email: <EMAIL>

package FixMyStreet::Geocode::OSM;

use warnings;
use strict;
use Data::Dumper;

use Digest::MD5 qw(md5_hex);
use Encode;
use File::Slurp;
use File::Path ();
use LWP::Simple qw($ua);
use Memcached;
use XML::Simple;
use mySociety::Locale;

my $osmapibase    = "http://www.openstreetmap.org/api/";
my $nominatimbase = "http://nominatim.openstreetmap.org/";

# string STRING CONTEXT
# Looks up on Nominatim, and caches, a user-inputted location.
# Returns array of (LAT, LON, ERROR), where ERROR is either undef, a string, or
# an array of matches if there are more than one. The information in the query
# may be used to disambiguate the location in cobranded versions of the site.
sub string {
    my ( $s, $c ) = @_;

    my $params = $c->cobrand->disambiguate_location($s);

    $s = FixMyStreet::Geocode::escape($s);
    $s .= '+' . $params->{town} if $params->{town} and $s !~ /$params->{town}/i;

    my $url = "${nominatimbase}search?";
    my %query_params = (
        q => $s,
        format => 'json',
	addressdetails => 1,
        #'accept-language' => '',
        email => 'info' . chr(64) . 'morus.se',
    );
    $query_params{viewbox} = $params->{bounds}[1] . ',' . $params->{bounds}[2] . ',' . $params->{bounds}[3] . ',' . $params->{bounds}[0]
        if $params->{bounds};
    $query_params{countrycodes} = $params->{country}
        if $params->{country};
    $url .= join('&', map { "$_=$query_params{$_}" } keys %query_params);

    my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'osm/';
    my $cache_file = $cache_dir . md5_hex($url);
    my $js;
    if (-s $cache_file) {
        $js = File::Slurp::read_file($cache_file);
    } else {
        $ua->timeout(15);
        $js = LWP::Simple::get($url);
        $js = encode_utf8($js) if utf8::is_utf8($js);
        File::Path::mkpath($cache_dir);
        File::Slurp::write_file($cache_file, $js) if $js;
    }

    if (!$js) {
        return { error => _('Sorry, we could not find that location.') };
    }

    $js = JSON->new->utf8->allow_nonref->decode($js);

    my ( $error, @valid_locations, $latitude, $longitude );
    foreach (@$js) {
        # These co-ordinates are output as query parameters in a URL, make sure they have a "."
	next unless	$_->{type} eq "town" ||
			$_->{type} eq "locality" ||
			$_->{type} eq "suburb" ||
			$_->{type} eq "city" ||
			$_->{type} eq "village" ||
			$_->{type} eq "hamlet" ||
			$_->{type} eq "secondary" ||
			$_->{type} eq "pedestrian" ||
			$_->{type} eq "tertiary" ||
			$_->{type} eq "square" ||
			$_->{type} eq "park" ||
			$_->{type} eq "primary" ||
			$_->{type} eq "unclassified" ||
			$_->{type} eq "residential";

	my $address = $_->{address}->{road}.", ".$_->{address}->{administrative};
	$address = $_->{address}->{locality}.", ".$_->{address}->{administrative} if $_->{type} eq "locality";
	$address = $_->{address}->{suburb}.", ".$_->{address}->{administrative} if $_->{type} eq "suburb";
	$address = $_->{address}->{town}.", ".$_->{address}->{state} if $_->{type} eq "town";
	$address = $_->{address}->{administrative}.", ".$_->{address}->{state} if $_->{type} eq "city";
	$address = $_->{address}->{village}.", ".$_->{address}->{administrative} if $_->{type} eq "village";
	$address = $_->{address}->{hamlet}.", ".$_->{address}->{state} if $_->{type} eq "hamlet";
	$address = $_->{address}->{leisure}.", ".$_->{address}->{administrative} if $_->{type} eq "square";
	$address = $_->{address}->{park}.", ".$_->{address}->{administrative} if $_->{type} eq "park";
	$address = $_->{address}->{pedestrian}.", ".$_->{address}->{administrative} if $_->{type} eq "pedestrian";

        ( $latitude, $longitude ) = ( $_->{lat}, $_->{lon} );
        mySociety::Locale::in_gb_locale {
            push (@$error, {
		address => $address,
                latitude => sprintf('%0.6f', $latitude),
                longitude => sprintf('%0.6f', $longitude)
            });
        };

        push (@valid_locations, $_);
    }

    my %seen;
    my @final_valid_locations = grep { $seen{$_->{address}}++ } @{$error};
open(OUT, '>', '/tmp/moo');
print OUT Dumper @valid_locations;
close OUT;
    return { latitude => $latitude, longitude => $longitude } if scalar @valid_locations == 1 or scalar keys %seen == 1;
    return { error => _('Sorry, we could not find that location.') } if scalar keys %seen == 0;
    return { error => $error };
}

sub reverse_geocode {
    my ($latitude, $longitude, $zoom) = @_;
    my $url =
    "${nominatimbase}reverse?format=xml&zoom=$zoom&lat=$latitude&lon=$longitude";
    my $key = "OSM:reverse_geocode:$url";
    my $result = Memcached::get($key);
    unless ($result) {
        my $j = LWP::Simple::get($url);
        if ($j) {
            Memcached::set($key, $j, 3600);
            my $ref = XMLin($j);
            return $ref;
        } else {
            print STDERR "No reply from $url\n";
        }
        return undef;
    }
    return XMLin($result);
}

sub _osmxml_to_hash {
    my ($xml, $type) = @_;
    my $ref = XMLin($xml);
    my %tags;
    if ('ARRAY' eq ref $ref->{$type}->{tag}) {
        map { $tags{$_->{'k'}} = $_->{'v'} } @{$ref->{$type}->{tag}};
        return \%tags;
    } else {
        return undef;
    }
}

sub get_object_tags {
    my ($type, $id) = @_;
    my $url = "${osmapibase}0.6/$type/$id";
    my $key = "OSM:get_object_tags:$url";
    my $result = Memcached::get($key);
    unless ($result) {
        my $j = LWP::Simple::get($url);
        if ($j) {
            Memcached::set($key, $j, 3600);
            return _osmxml_to_hash($j, $type);
        } else {
            print STDERR "No reply from $url\n";
        }
        return undef;
    }
    return _osmxml_to_hash($result, $type);
}

# A better alternative might be
# http://www.geonames.org/maps/osm-reverse-geocoder.html#findNearbyStreetsOSM
sub get_nearest_road_tags {
    my ( $cobrand, $latitude, $longitude ) = @_;
    my $inforef = reverse_geocode($latitude, $longitude, 16);
    if (exists $inforef->{result}->{osm_type}
        && 'way' eq $inforef->{result}->{osm_type}) {
        my $osmtags = get_object_tags('way',
                                      $inforef->{result}->{osm_id});
        unless ( exists $osmtags->{operator} ) {
            $osmtags->{operatorguess} = $cobrand->guess_road_operator( $osmtags );
        }
        return $osmtags;
    }
    return undef;
}

sub closest_road_text {
    my ( $cobrand, $latitude, $longitude ) = @_;
    my $str = '';
    my $osmtags = get_nearest_road_tags( $cobrand, $latitude, $longitude );
    if ($osmtags) {
        my ($name, $ref) = ('','');
        $name =  $osmtags->{name} if exists $osmtags->{name};
        $ref = " ($osmtags->{ref})" if exists $osmtags->{ref};
        if ($name || $ref) {
            $str .= _('The following information about the nearest road might be inaccurate or irrelevant, if the problem is close to several roads or close to a road without a name registered in OpenStreetMap.') . "\n\n";
            $str .= sprintf(_("Nearest named road to the pin placed on the map (automatically generated using OpenStreetMap): %s%s"),
                            $name, $ref) . "\n\n";

            if (my $operator = $osmtags->{operator}) {
                $str .= sprintf(_("Road operator for this named road (from OpenStreetMap): %s"),
                                $operator) . "\n\n";
            } elsif ($operator = $osmtags->{operatorguess}) {
                $str .= sprintf(_("Road operator for this named road (derived from road reference number and type): %s"),
                                $operator) . "\n\n";
            }
        }
    }
    return $str;
}

1;

<commit_msg>fixed search results Now, instead of selecting a potentially bad result, it shows a list of results hashed together using the first three words in display_name.<commit_after>#!/usr/bin/perl
#
# FixMyStreet:Geocode::OSM
# OpenStreetmap forward and reverse geocoding for FixMyStreet.
#
# Copyright (c) 2011 <NAME>. Some rights reserved.
# Email: <EMAIL>

package FixMyStreet::Geocode::OSM;

use warnings;
use strict;
use Data::Dumper;

use Digest::MD5 qw(md5_hex);
use Encode;
use File::Slurp;
use File::Path ();
use LWP::Simple qw($ua);
use Memcached;
use XML::Simple;
use mySociety::Locale;

my $osmapibase    = "http://www.openstreetmap.org/api/";
my $nominatimbase = "http://nominatim.openstreetmap.org/";

# string STRING CONTEXT
# Looks up on Nominatim, and caches, a user-inputted location.
# Returns array of (LAT, LON, ERROR), where ERROR is either undef, a string, or
# an array of matches if there are more than one. The information in the query
# may be used to disambiguate the location in cobranded versions of the site.
sub string {
    my ( $s, $c ) = @_;

    my $params = $c->cobrand->disambiguate_location($s);

    $s = FixMyStreet::Geocode::escape($s);
    # $s .= '+' . $params->{town} if $params->{town} and $s !~ /$params->{town}/i;

    my $url = "${nominatimbase}search?";
    my %query_params = (
        q => $s,
        format => 'json',
	addressdetails => 1,
	limit => 20,
        #'accept-language' => '',
        email => 'info' . chr(64) . 'morus.se',
    );
    # $query_params{viewbox} = $params->{bounds}[1] . ',' . $params->{bounds}[2] . ',' . $params->{bounds}[3] . ',' . $params->{bounds}[0]
    #     if $params->{bounds};
    $query_params{countrycodes} = $params->{country}
        if $params->{country};
    $url .= join('&', map { "$_=$query_params{$_}" } keys %query_params);

    my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'osm/';
    my $cache_file = $cache_dir . md5_hex($url);
    my $js;
    if (-s $cache_file) {
        $js = File::Slurp::read_file($cache_file);
    } else {
        $ua->timeout(15);
        $js = LWP::Simple::get($url);
        $js = encode_utf8($js) if utf8::is_utf8($js);
        File::Path::mkpath($cache_dir);
        File::Slurp::write_file($cache_file, $js) if $js;
    }

    if (!$js) {
        return { error => _('Sorry, we could not find that location.') };
    }

    $js = JSON->new->utf8->allow_nonref->decode($js);

    my ( %locations, $error, @valid_locations, $latitude, $longitude );
    foreach (@$js) {
        # These co-ordinates are output as query parameters in a URL, make sure they have a "."
	next if $_->{class} eq "boundary";

	my @s = split(/,/, $_->{display_name});

	my $address = join(",", @s[0,1,2]);

        $locations{$address} = [$_->{lat}, $_->{lon}];
    }

    my ($key) = keys %locations;

    return { latitude => $locations{$key}[0], longitude => $locations{$key}[1] } if scalar keys %locations == 1;
    return { error => _('Sorry, we could not find that location.') } if scalar keys %locations == 0;

    foreach $key (keys %locations) {
        ( $latitude, $longitude ) = ($locations{$key}[0], $locations{$key}[1]);
        mySociety::Locale::in_gb_locale {
            push (@$error, {
		address => $key,
                latitude => sprintf('%0.6f', $latitude),
                longitude => sprintf('%0.6f', $longitude)
            });
        };
    }

    return { error => $error };
}

sub reverse_geocode {
    my ($latitude, $longitude, $zoom) = @_;
    my $url =
    "${nominatimbase}reverse?format=xml&zoom=$zoom&lat=$latitude&lon=$longitude";
    my $key = "OSM:reverse_geocode:$url";
    my $result = Memcached::get($key);
    unless ($result) {
        my $j = LWP::Simple::get($url);
        if ($j) {
            Memcached::set($key, $j, 3600);
            my $ref = XMLin($j);
            return $ref;
        } else {
            print STDERR "No reply from $url\n";
        }
        return undef;
    }
    return XMLin($result);
}

sub _osmxml_to_hash {
    my ($xml, $type) = @_;
    my $ref = XMLin($xml);
    my %tags;
    if ('ARRAY' eq ref $ref->{$type}->{tag}) {
        map { $tags{$_->{'k'}} = $_->{'v'} } @{$ref->{$type}->{tag}};
        return \%tags;
    } else {
        return undef;
    }
}

sub get_object_tags {
    my ($type, $id) = @_;
    my $url = "${osmapibase}0.6/$type/$id";
    my $key = "OSM:get_object_tags:$url";
    my $result = Memcached::get($key);
    unless ($result) {
        my $j = LWP::Simple::get($url);
        if ($j) {
            Memcached::set($key, $j, 3600);
            return _osmxml_to_hash($j, $type);
        } else {
            print STDERR "No reply from $url\n";
        }
        return undef;
    }
    return _osmxml_to_hash($result, $type);
}

# A better alternative might be
# http://www.geonames.org/maps/osm-reverse-geocoder.html#findNearbyStreetsOSM
sub get_nearest_road_tags {
    my ( $cobrand, $latitude, $longitude ) = @_;
    my $inforef = reverse_geocode($latitude, $longitude, 16);
    if (exists $inforef->{result}->{osm_type}
        && 'way' eq $inforef->{result}->{osm_type}) {
        my $osmtags = get_object_tags('way',
                                      $inforef->{result}->{osm_id});
        unless ( exists $osmtags->{operator} ) {
            $osmtags->{operatorguess} = $cobrand->guess_road_operator( $osmtags );
        }
        return $osmtags;
    }
    return undef;
}

sub closest_road_text {
    my ( $cobrand, $latitude, $longitude ) = @_;
    my $str = '';
    my $osmtags = get_nearest_road_tags( $cobrand, $latitude, $longitude );
    if ($osmtags) {
        my ($name, $ref) = ('','');
        $name =  $osmtags->{name} if exists $osmtags->{name};
        $ref = " ($osmtags->{ref})" if exists $osmtags->{ref};
        if ($name || $ref) {
            $str .= _('The following information about the nearest road might be inaccurate or irrelevant, if the problem is close to several roads or close to a road without a name registered in OpenStreetMap.') . "\n\n";
            $str .= sprintf(_("Nearest named road to the pin placed on the map (automatically generated using OpenStreetMap): %s%s"),
                            $name, $ref) . "\n\n";

            if (my $operator = $osmtags->{operator}) {
                $str .= sprintf(_("Road operator for this named road (from OpenStreetMap): %s"),
                                $operator) . "\n\n";
            } elsif ($operator = $osmtags->{operatorguess}) {
                $str .= sprintf(_("Road operator for this named road (derived from road reference number and type): %s"),
                                $operator) . "\n\n";
            }
        }
    }
    return $str;
}

1;


{'fraction_non_alphanumeric': 0.18923384043505198, 'fraction_numerical': 0.0060576856887175606, 'mean_word_length': 2.6484178804620795, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 183, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7039369', 'n_tokens_mistral': 5214, 'n_tokens_neox': 5017, 'n_words': 1230}