<commit_before>import json
import re

from rest_framework import serializers
from rest_framework import viewsets
from rest_framework.settings import api_settings

from rest_framework_jsonp.renderers import JSONPRenderer

from django_filters import rest_framework as filters

from froide.helper.api_utils import OpenRefineReconciliationMixin

from .models import GeoRegion


GERMAN_PLZ_RE = re.compile('\d{5}')


class GeoRegionSerializer(serializers.HyperlinkedModelSerializer):
    resource_uri = serializers.HyperlinkedIdentityField(
        view_name='api:georegion-detail',
        lookup_field='pk'
    )
    part_of = serializers.HyperlinkedRelatedField(
        view_name='api:georegion-detail',
        lookup_field='pk',
        read_only=True,
        many=False
    )
    geom = serializers.SerializerMethodField()
    gov_seat = serializers.SerializerMethodField()
    centroid = serializers.SerializerMethodField()

    class Meta:
        model = GeoRegion
        depth = 0
        fields = (
            'resource_uri', 'id', 'name', 'slug', 'kind',
            'kind_detail', 'level',
            'region_identifier', 'global_identifier',
            'area', 'population', 'valid_on',
            'geom', 'gov_seat',
            'centroid',
            'part_of',
        )

    def get_geom(self, obj):
        if obj.geom is not None:
            return json.loads(obj.geom.json)
        return None

    def get_gov_seat(self, obj):
        if obj.gov_seat is not None:
            return json.loads(obj.gov_seat.json)
        return None

    def get_centroid(self, obj):
        if obj.geom is not None:
            return json.loads(obj.geom.centroid.json)
        return None


class GeoRegionFilter(filters.FilterSet):
    q = filters.CharFilter(method='search_filter')
    kind = filters.CharFilter(method='kind_filter')
    level = filters.NumberFilter(method='level_filter')

    class Meta:
        model = GeoRegion
        fields = (
            'name', 'level', 'kind', 'slug'
        )

    def search_filter(self, queryset, name, value):
        return queryset.filter(name__icontains=value)

    def kind_filter(self, queryset, name, value):
        return queryset.filter(kind=value)

    def level_filter(self, queryset, name, value):
        return queryset.filter(level=value)


class GeoRegionViewSet(OpenRefineReconciliationMixin,
                       viewsets.ReadOnlyModelViewSet):
    serializer_class = GeoRegionSerializer
    queryset = GeoRegion.objects.all()
    filter_backends = (filters.DjangoFilterBackend,)
    filterset_class = GeoRegionFilter

    # OpenRefine needs JSONP responses
    # This is OK because authentication is not considered
    renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (JSONPRenderer,)

    class RECONCILIATION_META:
        name = 'GeoRegion'
        id = 'georegion'
        model = GeoRegion
        api_list = 'api:georegion-list'
        obj_short_link = None
        filters = ['kind', 'level']
        properties = [{
            'id': 'population',
            'name': 'population',
            }, {
            'id': 'area',
            'name': 'area',
            }, {
            'id': 'geom',
            'name': 'geom',
            }, {
            'id': 'name',
            'name': 'Name'
            }, {
            'id': 'id',
            'name': 'ID'
            }, {
            'id': 'slug',
            'name': 'Slug'
            }, {
            'id': 'kind',
            'name': 'Kind'
        }]
        properties_dict = {
            p['id']: p for p in properties
        }

    def _search_reconciliation_results(self, query, filters, limit):
        qs = GeoRegion.objects.all()
        for key, val in filters.items():
            qs = qs.filter(**{key: val})
        match = GERMAN_PLZ_RE.match(query)
        zip_region = None
        if match:
            try:
                zip_region = GeoRegion.objects.get(name=query, kind='zipcode')
                qs = qs.filter(geom__covers=zip_region.geom.centroid)
            except GeoRegion.DoesNotExist:
                pass

        if not match or not zip_region:
            qs = qs.filter(name__contains=query)[:limit]

        for r in qs:
            yield {
                'id': str(r.pk),
                'name': r.name,
                'type': ['georegion'],
                'score': 4,
                'match': True  # FIXME: this is quite arbitrary
            }
<commit_msg>Add identifiers to georegion reconciliation props<commit_after>import json
import re

from rest_framework import serializers
from rest_framework import viewsets
from rest_framework.settings import api_settings

from rest_framework_jsonp.renderers import JSONPRenderer

from django_filters import rest_framework as filters

from froide.helper.api_utils import OpenRefineReconciliationMixin

from .models import GeoRegion


GERMAN_PLZ_RE = re.compile('\d{5}')


class GeoRegionSerializer(serializers.HyperlinkedModelSerializer):
    resource_uri = serializers.HyperlinkedIdentityField(
        view_name='api:georegion-detail',
        lookup_field='pk'
    )
    part_of = serializers.HyperlinkedRelatedField(
        view_name='api:georegion-detail',
        lookup_field='pk',
        read_only=True,
        many=False
    )
    geom = serializers.SerializerMethodField()
    gov_seat = serializers.SerializerMethodField()
    centroid = serializers.SerializerMethodField()

    class Meta:
        model = GeoRegion
        depth = 0
        fields = (
            'resource_uri', 'id', 'name', 'slug', 'kind',
            'kind_detail', 'level',
            'region_identifier', 'global_identifier',
            'area', 'population', 'valid_on',
            'geom', 'gov_seat',
            'centroid',
            'part_of',
        )

    def get_geom(self, obj):
        if obj.geom is not None:
            return json.loads(obj.geom.json)
        return None

    def get_gov_seat(self, obj):
        if obj.gov_seat is not None:
            return json.loads(obj.gov_seat.json)
        return None

    def get_centroid(self, obj):
        if obj.geom is not None:
            return json.loads(obj.geom.centroid.json)
        return None


class GeoRegionFilter(filters.FilterSet):
    q = filters.CharFilter(method='search_filter')
    kind = filters.CharFilter(method='kind_filter')
    level = filters.NumberFilter(method='level_filter')

    class Meta:
        model = GeoRegion
        fields = (
            'name', 'level', 'kind', 'slug'
        )

    def search_filter(self, queryset, name, value):
        return queryset.filter(name__icontains=value)

    def kind_filter(self, queryset, name, value):
        return queryset.filter(kind=value)

    def level_filter(self, queryset, name, value):
        return queryset.filter(level=value)


class GeoRegionViewSet(OpenRefineReconciliationMixin,
                       viewsets.ReadOnlyModelViewSet):
    serializer_class = GeoRegionSerializer
    queryset = GeoRegion.objects.all()
    filter_backends = (filters.DjangoFilterBackend,)
    filterset_class = GeoRegionFilter

    # OpenRefine needs JSONP responses
    # This is OK because authentication is not considered
    renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (JSONPRenderer,)

    class RECONCILIATION_META:
        name = 'GeoRegion'
        id = 'georegion'
        model = GeoRegion
        api_list = 'api:georegion-list'
        obj_short_link = None
        filters = ['kind', 'level']
        properties = [{
            'id': 'population',
            'name': 'population',
            }, {
            'id': 'area',
            'name': 'area',
            }, {
            'id': 'geom',
            'name': 'geom',
            }, {
            'id': 'name',
            'name': 'Name'
            }, {
            'id': 'id',
            'name': 'ID'
            }, {
            'id': 'slug',
            'name': 'Slug'
            }, {
            'id': 'kind',
            'name': 'Kind'
            }, {
            'id': 'region_identifier',
            'name': 'Region identifier'
            }, {
            'id': 'global_identifier',
            'name': 'Global identifier'
        }]
        properties_dict = {
            p['id']: p for p in properties
        }

    def _search_reconciliation_results(self, query, filters, limit):
        qs = GeoRegion.objects.all()
        for key, val in filters.items():
            qs = qs.filter(**{key: val})
        match = GERMAN_PLZ_RE.match(query)
        zip_region = None
        if match:
            try:
                zip_region = GeoRegion.objects.get(name=query, kind='zipcode')
                qs = qs.filter(geom__covers=zip_region.geom.centroid)
            except GeoRegion.DoesNotExist:
                pass

        if not match or not zip_region:
            qs = qs.filter(name__contains=query)[:limit]

        for r in qs:
            yield {
                'id': str(r.pk),
                'name': r.name,
                'type': ['georegion'],
                'score': 4,
                'match': True  # FIXME: this is quite arbitrary
            }

{'fraction_non_alphanumeric': 0.10648047130700415, 'fraction_numerical': 0.0006545930613135501, 'mean_word_length': 2.0495675316034596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5691296', 'n_tokens_mistral': 2755, 'n_tokens_neox': 2642, 'n_words': 587}