<commit_before># -:- encoding: UTF8 -:-
# Copyright 2011 Omniscale (http://omniscale.com)
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import imposm.geom

ANY = '__any__'

__all__ = [
    'LineStrings',
    'Polygons',
    'Points',
    'Options',
    'PolygonTable',
    'ZOrder',
    'PointTable',
    'String',
    'LineStringTable',
    'Direction',
    'OneOfInt',
    'Integer',
    'WayZOrder',
    'Bool',
    'GeneralizedTable',
    'UnionView',
]


class Mapping(object):
    table = None
    fields = ()
    field_filter = ()
    classname = None
    _insert_stmt = None
    
    filter_out_names = set((
        'fixme', 'fix me', 'fix-me!',
        'kein name', 'kein',
        'unbenannt', 'unbekannt',
        'noch unbekannt', 'noch ohne namen',
        'noname', 'unnamed', 'namenlos', 'no_name', 'no name',
    ))
    
    def __init__(self, name, mapping, fields=None, field_filter=None):
        self.name = name
        self.mapping = mapping
        if fields:
            self.fields = fields
        if field_filter:
            self.field_filter = field_filter
        
    @property
    def insert_stmt(self):
        if not self._insert_stmt:
            self._insert_stmt = self.table('osm_' + self.name, self).insert_stmt
        return self._insert_stmt
    
    def name_filter(self, osm_elem):
        if osm_elem.name and osm_elem.name.lower() in self.filter_out_names:
            osm_elem.name = ''
    
    def extra_field_names(self):
        return [n for n,_ in self.fields] + [n for n,_ in self.field_filter]
    
    def build_geom(self, osm_elem):
        try:
            geom = self.geom_builder.build_checked_geom(osm_elem)
            osm_elem.geom = geom
        except imposm.geom.InvalidGeometryError, ex:
            raise DropElem('invalid geometry: %s' % (ex, ))
    
    def field_values(self, osm_elem):
        return [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields]
    
    def filter(self, osm_elem):
        [t.filter(osm_elem.tags.get(n), osm_elem) for n, t in self.field_filter]
    
    def __repr__(self):
        return '<Mapping for %s>' % self.name


class TagMapper(object):
    def __init__(self, mappings):
        self.mappings = mappings
        self._init_map()

    def _init_map(self):
        self.points = {}
        self.lines = {}
        self.polygons = {}
        self.nodes_tags = set()
        self.ways_tags = set()

        for mapping in self.mappings:
            if mapping.table is PointTable:
                tags = self.nodes_tags
                add_to = self.points
            elif mapping.table is LineStringTable:
                tags = self.ways_tags
                add_to = self.lines
            elif mapping.table is PolygonTable:
                tags = self.ways_tags
                add_to = self.polygons

            tags.update(mapping.extra_field_names())

            for tag, types in mapping.mapping.iteritems():
                add_to.setdefault(tag, {})
                tags.add(tag)
                for type in types:
                    add_to[tag].setdefault(type, [])
                    add_to[tag][type].append(mapping)

    def for_nodes(self, tags):
        return self._mapping_for_tags(self.points, tags)

    def for_ways(self, tags):
        return (self._mapping_for_tags(self.lines, tags) + 
                self._mapping_for_tags(self.polygons, tags))

    def for_relations(self, tags):
        return self._mapping_for_tags(self.polygons, tags)

    def _tag_filter(self, filter_tags):
        filter_tags['name'] = {'__any__': []}
        def filter(tags):
            for k in tags.keys():
                if k not in filter_tags:
                    del tags[k]
                else:
                    if tags[k] not in filter_tags[k] and '__any__' not in filter_tags[k]:
                        del tags[k]
            if 'name' in tags and len(tags) == 1:
                del tags['name']
        return filter

    def tag_filter_for_nodes(self):
        tags = dict(self.points)
        return self._tag_filter(tags)

    def tag_filter_for_ways(self):
        tags = dict(self.lines)
        tags.update(self.polygons)
        return self._tag_filter(tags)

    def tag_filter_for_relations(self):
        relation_tags = dict(self.lines)
        relation_tags.update(self.polygons)
        relation_tags['type'] = 'multipolygon'  # for type=multipolygon
        _rel_filter = self._tag_filter(relation_tags)
        def rel_filter(tags):
            if tags.get('type') != 'multipolygon':
                tags.clear()
                return
            _rel_filter(tags)
        return rel_filter

    def _mapping_for_tags(self, tag_map, tags):
        result = []
        mapping_set = set()

        for tag_name in tags:
            if tag_name in tag_map:
                tag_value = tags[tag_name]
                mappings = []
                if tag_value in tag_map[tag_name]:
                    mappings.extend(tag_map[tag_name][tag_value])
                elif ANY in tag_map[tag_name]:
                    mappings.extend(tag_map[tag_name][ANY])

                new_mappings = []
                for proc in mappings:
                    if proc not in mapping_set:
                        mapping_set.add(proc)
                        new_mappings.append(proc)
                if new_mappings:
                    result.append(((tag_name, tag_value), tuple(new_mappings)))
<commit_msg>fixed merging of tag mappings<commit_after># -:- encoding: UTF8 -:-
# Copyright 2011 Omniscale (http://omniscale.com)
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import imposm.geom

ANY = '__any__'

__all__ = [
    'LineStrings',
    'Polygons',
    'Points',
    'Options',
    'PolygonTable',
    'ZOrder',
    'PointTable',
    'String',
    'LineStringTable',
    'Direction',
    'OneOfInt',
    'Integer',
    'WayZOrder',
    'Bool',
    'GeneralizedTable',
    'UnionView',
]


class Mapping(object):
    table = None
    fields = ()
    field_filter = ()
    classname = None
    _insert_stmt = None
    
    filter_out_names = set((
        'fixme', 'fix me', 'fix-me!',
        'kein name', 'kein',
        'unbenannt', 'unbekannt',
        'noch unbekannt', 'noch ohne namen',
        'noname', 'unnamed', 'namenlos', 'no_name', 'no name',
    ))
    
    def __init__(self, name, mapping, fields=None, field_filter=None):
        self.name = name
        self.mapping = mapping
        if fields:
            self.fields = fields
        if field_filter:
            self.field_filter = field_filter
        
    @property
    def insert_stmt(self):
        if not self._insert_stmt:
            self._insert_stmt = self.table('osm_' + self.name, self).insert_stmt
        return self._insert_stmt
    
    def name_filter(self, osm_elem):
        if osm_elem.name and osm_elem.name.lower() in self.filter_out_names:
            osm_elem.name = ''
    
    def extra_field_names(self):
        return [n for n,_ in self.fields] + [n for n,_ in self.field_filter]
    
    def build_geom(self, osm_elem):
        try:
            geom = self.geom_builder.build_checked_geom(osm_elem)
            osm_elem.geom = geom
        except imposm.geom.InvalidGeometryError, ex:
            raise DropElem('invalid geometry: %s' % (ex, ))
    
    def field_values(self, osm_elem):
        return [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields]
    
    def filter(self, osm_elem):
        [t.filter(osm_elem.tags.get(n), osm_elem) for n, t in self.field_filter]
    
    def __repr__(self):
        return '<Mapping for %s>' % self.name


class TagMapper(object):
    def __init__(self, mappings):
        self.mappings = mappings
        self._init_map()

    def _init_map(self):
        self.points = {}
        self.lines = {}
        self.polygons = {}
        self.nodes_tags = set()
        self.ways_tags = set()

        for mapping in self.mappings:
            if mapping.table is PointTable:
                tags = self.nodes_tags
                add_to = self.points
            elif mapping.table is LineStringTable:
                tags = self.ways_tags
                add_to = self.lines
            elif mapping.table is PolygonTable:
                tags = self.ways_tags
                add_to = self.polygons

            tags.update(mapping.extra_field_names())

            for tag, types in mapping.mapping.iteritems():
                add_to.setdefault(tag, {})
                tags.add(tag)
                for type in types:
                    add_to[tag].setdefault(type, [])
                    add_to[tag][type].append(mapping)

    def for_nodes(self, tags):
        return self._mapping_for_tags(self.points, tags)

    def for_ways(self, tags):
        return (self._mapping_for_tags(self.lines, tags) + 
                self._mapping_for_tags(self.polygons, tags))

    def for_relations(self, tags):
        return self._mapping_for_tags(self.polygons, tags)

    def _tag_filter(self, filter_tags):
        filter_tags['name'] = {'__any__': []}
        def filter(tags):
            for k in tags.keys():
                if k not in filter_tags:
                    del tags[k]
                else:
                    if tags[k] not in filter_tags[k] and '__any__' not in filter_tags[k]:
                        del tags[k]
            if 'name' in tags and len(tags) == 1:
                del tags['name']
        return filter

    def tag_filter_for_nodes(self):
        tags = dict(self.points)
        return self._tag_filter(tags)

    def tag_filter_for_ways(self):
        tags = dict()
        for k, v in self.lines.iteritems():
            tags.setdefault(k, {}).update(v)
        
        for k, v in self.polygons.iteritems():
            tags.setdefault(k, {}).update(v)
        return self._tag_filter(tags)

    def tag_filter_for_relations(self):
        tags = dict()
        for k, v in self.lines.iteritems():
            tags.setdefault(k, {}).update(v)
        for k, v in self.polygons.iteritems():
            tags.setdefault(k, {}).update(v)
        tags['type'] = 'multipolygon'  # for type=multipolygon
        _rel_filter = self._tag_filter(tags)
        def rel_filter(tags):
            if tags.get('type') != 'multipolygon':
                tags.clear()
                return
            _rel_filter(tags)
        return rel_filter

    def _mapping_for_tags(self, tag_map, tags):
        result = []
        mapping_set = set()

        for tag_name in tags:
            if tag_name in tag_map:
                tag_value = tags[tag_name]
                mappings = []
                if tag_value in tag_map[tag_name]:
                    mappings.extend(tag_map[tag_name][tag_value])
                elif ANY in tag_map[tag_name]:
                    mappings.extend(tag_map[tag_name][ANY])

                new_mappings = []
                for proc in mappings:
                    if proc not in mapping_set:
                        mapping_set.add(proc)
                        new_mappings.append(proc)
                if new_mappings:
                    result.append(((tag_name, tag_value), tuple(new_mappings)))
        
{'fraction_non_alphanumeric': 0.10351898941842343, 'fraction_numerical': 0.001640554507423509, 'mean_word_length': 1.9307692307692308, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5065384', 'n_tokens_mistral': 3768, 'n_tokens_neox': 3633, 'n_words': 900}