<commit_before># Copyright 2011 Omniscale (http://omniscale.com)
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import division
import os
import time

from imposm.geom import (
    PolygonBuilder,
    LineStringBuilder,
    InvalidGeometryError,
    IncompletePolygonError,
)
from imposm.merge import merge

import imposm.base
import imposm.geom
import shapely.geometry
import shapely.ops
import shapely.geos
import shapely.prepared


import logging
log = logging.getLogger(__name__)

IMPOSM_MULTIPOLYGON_REPORT = float(os.environ.get('IMPOSM_MULTIPOLYGON_REPORT', 60))
IMPOSM_MULTIPOLYGON_MAX_RING = int(os.environ.get('IMPOSM_MULTIPOLYGON_MAX_RING', 1000))

class RelationBuilder(object):
    def __init__(self, relation, ways_cache, coords_cache):
        self.relation = relation
        self.polygon_builder = PolygonBuilder()
        self.linestring_builder = LineStringBuilder()
        self.ways_cache = ways_cache
        self.coords_cache = coords_cache
    
    def fetch_ways(self):
        ways = []
        for member in self.relation.members:
            # skip label nodes, relations of relations, etc
            if member[1] != 'way': continue
            way = self.ways_cache.get(member[0])
            if way is None:
                log.debug('way not found %s:%s', self.relation.osm_id, member[0])
                raise IncompletePolygonError('way not found %s:%s' % (self.relation.osm_id, member[0]))
            if way.partial_refs:
                log.warn('multiple linestrings in way %s (relation %s)',
                       member[0], self.relation.osm_id)
                raise IncompletePolygonError()
            
            way.coords = self.fetch_way_coords(way)
            ways.append(way)
        return ways
    
    def build_rings(self, ways):
        rings = []
        incomplete_rings = []
        
        for ring in (Ring(w) for w in ways):
            if ring.is_closed():
                ring.geom = self.polygon_builder.build_checked_geom(ring, validate=True)
                rings.append(ring)
            else:
                incomplete_rings.append(ring)
        
        merged_rings = self.build_ring_from_incomplete(incomplete_rings)
        
        return rings + merged_rings
        
    def build_ring_from_incomplete(self, incomplete_rings):
        
        rings = merge_rings(incomplete_rings)
        for ring in rings:
            if not ring.is_closed():
                raise InvalidGeometryError('linestrings from relation %s do not form a ring' %
                        self.relation.osm_id)
            
            ring.geom = self.polygon_builder.build_checked_geom(ring, validate=True)
        return rings
    
    def fetch_way_coords(self, way):
        """
        Fetch all coordinates of way.refs.
        """
        coords = self.coords_cache.get_coords(way.refs)
        if coords is None:
            log.debug('missing coord from way %s in relation %s',
                way.osm_id, self.relation.osm_id)
            raise IncompletePolygonError()
        
        return coords
    
    def build_relation_geometry(self, rings):
        """
        Build relation geometry from rings.
        """
        rings.sort(key=lambda x: x.geom.area, reverse=True)
        
        # add/subtract all rings from largest
        polygon = rings.pop(0)
        polygon.inserted = True
        rel_tags = relation_tags(self.relation.tags, polygon.tags)
        
        geom = polygon.geom
        for r in rings:
            if geom.contains(r.geom):
                # inside -> hole -> subtract
                geom = geom.difference(r.geom)
                if tags_same_or_empty(rel_tags, r.tags):
                    r.inserted = True
                else:
                    r.inserted = False
            else:
                # outside or overlap -> merge(union) to multipolygon or to polygon
                try:
                    geom = geom.union(r.geom)
                except shapely.geos.TopologicalError:
                    raise InvalidGeometryError('multipolygon relation (%s) result is invalid'
                                               ' (topological error)' % self.relation.osm_id)
                r.inserted = True
        if not geom.is_valid:
            raise InvalidGeometryError('multipolygon relation (%s) result is invalid' %
                                       self.relation.osm_id)
        
        self.relation.geom = geom
        self.relation.tags = rel_tags
        all_ways = polygon.ways
        for r in rings:
            all_ways.extend(r.ways)
        self.relation.ways = all_ways
    
    def mark_inserted_ways(self, inserted_ways_queue):
        for w in self.relation.ways:
            if w.inserted:
                inserted_ways_queue.put(w.osm_id)
    
    def build(self):
        try:
            time_start = time.time()
            ways = self.fetch_ways()
            time_ways = time.time() - time_start
            if not ways:
                raise IncompletePolygonError('no ways found')
            time_start = time.time()
            rings = self.build_rings(ways)
            time_rings = time.time() - time_start

            if len(rings) > IMPOSM_MULTIPOLYGON_MAX_RING:
                log.warn('skipping relation %d with %d ways (%.1fms) and %d rings (%.1fms): too many rings',
                    self.relation.osm_id, len(ways), time_ways*1000, len(rings), time_rings*1000)
            
            time_start = time.time()
            self.build_relation_geometry(rings)
            time_relations = time.time() - time_start
            
            if time_ways + time_rings + time_relations > IMPOSM_MULTIPOLYGON_REPORT:
                log.warn('building relation %d with %d ways (%.1fms) and %d rings (%.1fms) took %.1fms',
                    self.relation.osm_id, len(ways), time_ways*1000, len(rings), time_rings*1000, time_relations*1000)
        except InvalidGeometryError, ex:
            log.debug(ex)
            raise IncompletePolygonError(ex)
        except IncompletePolygonError:
            raise
        except Exception, ex:
            log.warn('error while building multipolygon:')
            log.exception(ex)
            raise IncompletePolygonError(ex)

def relation_tags(rel_tags, way_tags):
    result = dict(rel_tags)
    
    if 'type' in result: del result['type']
    if 'name' in result: del result['name']
    
    if not result:
        # use way_tags
        result.update(way_tags)
    else:
        if 'name' in rel_tags:
            # put back name
            result['name'] = rel_tags['name']
        <commit_msg>fixed skipping of too large multipolygons. closes #8<commit_after># Copyright 2011 Omniscale (http://omniscale.com)
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import division
import os
import time

from imposm.geom import (
    PolygonBuilder,
    LineStringBuilder,
    InvalidGeometryError,
    IncompletePolygonError,
)
from imposm.merge import merge

import imposm.base
import imposm.geom
import shapely.geometry
import shapely.ops
import shapely.geos
import shapely.prepared


import logging
log = logging.getLogger(__name__)

IMPOSM_MULTIPOLYGON_REPORT = float(os.environ.get('IMPOSM_MULTIPOLYGON_REPORT', 60))
IMPOSM_MULTIPOLYGON_MAX_RING = int(os.environ.get('IMPOSM_MULTIPOLYGON_MAX_RING', 1000))

class RelationBuilder(object):
    def __init__(self, relation, ways_cache, coords_cache):
        self.relation = relation
        self.polygon_builder = PolygonBuilder()
        self.linestring_builder = LineStringBuilder()
        self.ways_cache = ways_cache
        self.coords_cache = coords_cache
    
    def fetch_ways(self):
        ways = []
        for member in self.relation.members:
            # skip label nodes, relations of relations, etc
            if member[1] != 'way': continue
            way = self.ways_cache.get(member[0])
            if way is None:
                log.debug('way not found %s:%s', self.relation.osm_id, member[0])
                raise IncompletePolygonError('way not found %s:%s' % (self.relation.osm_id, member[0]))
            if way.partial_refs:
                log.warn('multiple linestrings in way %s (relation %s)',
                       member[0], self.relation.osm_id)
                raise IncompletePolygonError()
            
            way.coords = self.fetch_way_coords(way)
            ways.append(way)
        return ways
    
    def build_rings(self, ways):
        rings = []
        incomplete_rings = []
        
        for ring in (Ring(w) for w in ways):
            if ring.is_closed():
                ring.geom = self.polygon_builder.build_checked_geom(ring, validate=True)
                rings.append(ring)
            else:
                incomplete_rings.append(ring)
        
        merged_rings = self.build_ring_from_incomplete(incomplete_rings)
        
        return rings + merged_rings
        
    def build_ring_from_incomplete(self, incomplete_rings):
        
        rings = merge_rings(incomplete_rings)
        for ring in rings:
            if not ring.is_closed():
                raise InvalidGeometryError('linestrings from relation %s do not form a ring' %
                        self.relation.osm_id)
            
            ring.geom = self.polygon_builder.build_checked_geom(ring, validate=True)
        return rings
    
    def fetch_way_coords(self, way):
        """
        Fetch all coordinates of way.refs.
        """
        coords = self.coords_cache.get_coords(way.refs)
        if coords is None:
            log.debug('missing coord from way %s in relation %s',
                way.osm_id, self.relation.osm_id)
            raise IncompletePolygonError()
        
        return coords
    
    def build_relation_geometry(self, rings):
        """
        Build relation geometry from rings.
        """
        rings.sort(key=lambda x: x.geom.area, reverse=True)
        
        # add/subtract all rings from largest
        polygon = rings.pop(0)
        polygon.inserted = True
        rel_tags = relation_tags(self.relation.tags, polygon.tags)
        
        geom = polygon.geom
        for r in rings:
            if geom.contains(r.geom):
                # inside -> hole -> subtract
                geom = geom.difference(r.geom)
                if tags_same_or_empty(rel_tags, r.tags):
                    r.inserted = True
                else:
                    r.inserted = False
            else:
                # outside or overlap -> merge(union) to multipolygon or to polygon
                try:
                    geom = geom.union(r.geom)
                except shapely.geos.TopologicalError:
                    raise InvalidGeometryError('multipolygon relation (%s) result is invalid'
                                               ' (topological error)' % self.relation.osm_id)
                r.inserted = True
        if not geom.is_valid:
            raise InvalidGeometryError('multipolygon relation (%s) result is invalid' %
                                       self.relation.osm_id)
        
        self.relation.geom = geom
        self.relation.tags = rel_tags
        all_ways = polygon.ways
        for r in rings:
            all_ways.extend(r.ways)
        self.relation.ways = all_ways
    
    def mark_inserted_ways(self, inserted_ways_queue):
        for w in self.relation.ways:
            if w.inserted:
                inserted_ways_queue.put(w.osm_id)
    
    def build(self):
        try:
            time_start = time.time()
            ways = self.fetch_ways()
            time_ways = time.time() - time_start
            if not ways:
                raise IncompletePolygonError('no ways found')
            time_start = time.time()
            rings = self.build_rings(ways)
            time_rings = time.time() - time_start

            if len(rings) > IMPOSM_MULTIPOLYGON_MAX_RING:
                log.warn('skipping relation %d with %d ways (%.1fms) and %d rings (%.1fms): too many rings',
                    self.relation.osm_id, len(ways), time_ways*1000, len(rings), time_rings*1000)
                raise IncompletePolygonError('skipping too large multipolygon')
            time_start = time.time()
            self.build_relation_geometry(rings)
            time_relations = time.time() - time_start
            
            if time_ways + time_rings + time_relations > IMPOSM_MULTIPOLYGON_REPORT:
                log.warn('building relation %d with %d ways (%.1fms) and %d rings (%.1fms) took %.1fms',
                    self.relation.osm_id, len(ways), time_ways*1000, len(rings), time_rings*1000, time_relations*1000)
        except InvalidGeometryError, ex:
            log.debug(ex)
            raise IncompletePolygonError(ex)
        except IncompletePolygonError:
            raise
        except Exception, ex:
            log.warn('error while building multipolygon:')
            log.exception(ex)
            raise IncompletePolygonError(ex)

def relation_tags(rel_tags, way_tags):
    result = dict(rel_tags)
    
    if 'type' in result: del result['type']
    if 'name' in result: del result['name']
    
    if not result:
        # use way_tags
        result.update(way_tags)
    else:
        if 'name' in rel_tags:
            # put back name
            result['name'] = rel_tags['name']
        
{'fraction_non_alphanumeric': 0.08448155880306193, 'fraction_numerical': 0.006332637439109255, 'mean_word_length': 1.9612610756233257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '78909', 'n_tokens_mistral': 4225, 'n_tokens_neox': 4053, 'n_words': 1036}