<commit_before>import json
import os

import datetime

from mimetypes import guess_type
from urllib.parse import urlparse

from django.conf import settings
from django.contrib.gis.geos import Point
from django.core.files.base import ContentFile
from django.db import models
from django.utils.translation import gettext as _
from django.core.files.uploadedfile import UploadedFile

from geotrek.common.parsers import (AttachmentParserMixin, Parser,
                                    TourInSoftParser)
from geotrek.tourism.models import (InformationDesk, TouristicContent, TouristicEvent,
                                    TouristicContentType1, TouristicContentType2)


class TouristicContentMixin:
    # Mixin which handle multiple type1/2 with the same name in different categories
    def get_to_delete_kwargs(self):
        # FIXME: use mapping if it exists
        kwargs = {}
        for dst, val in self.constant_fields.items():
            field = self.model._meta.get_field(dst)
            if isinstance(field, models.ForeignKey):
                natural_key = self.natural_keys[dst]
                try:
                    kwargs[dst] = field.remote_field.model.objects.get(**{natural_key: val})
                except field.remote_field.model.DoesNotExist:
                    return None
            else:
                kwargs[dst] = val
        for dst, val in self.m2m_constant_fields.items():
            assert not self.separator or self.separator not in val
            field = self.model._meta.get_field(dst)
            natural_key = self.natural_keys[dst]
            filters = {natural_key: subval for subval in val}
            if not filters:
                continue
            if dst in ('type1', 'type2'):
                filters['category'] = kwargs['category'].pk
            try:
                kwargs[dst] = field.remote_field.model.objects.get(**filters)
            except field.remote_field.model.DoesNotExist:
                return None
        return kwargs


class ApidaeParser(AttachmentParserMixin, Parser):
    """Parser to import "anything" from APIDAE"""
    separator = None
    api_key = None
    project_id = None
    selection_id = None
    url = 'http://api.apidae-tourisme.com/api/v002/recherche/list-objets-touristiques/'
    model = None
    eid = 'eid'
    fields = {
        'name': 'nom.libelleFr',
    }
    constant_fields = {
        'published': True,
    }
    non_fields = {
        'attachments': 'illustrations',
    }
    # Use for foreign keys. When the key is a foreign key, it will try to get the key's value.
    # In django : 'category' : 'label' -> category.label
    field_options = {
        'name': {'required': True},
        'geom': {'required': True},
    }
    size = 100
    skip = 0
    responseFields = [
        'id',
        'nom',
        'presentation.descriptifCourt',
        'presentation.descriptifDetaille',
        'localisation.adresse',
        'localisation.geolocalisation.geoJson.coordinates',
        'localisation.geolocalisation.complement.libelleFr',
        'informations.moyensCommunication',
        'ouverture.periodeEnClair',
        'informationsHebergementCollectif.capacite.capaciteTotale',
        'informationsHebergementCollectif.hebergementCollectifType.libelleFr',
        'descriptionTarif.tarifsEnClair',
        'descriptionTarif.modesPaiement',
        'prestations.services',
        'gestion.dateModification',
        'gestion.membreProprietaire.nom',
        'illustrations'
    ]

    @property
    def items(self):
        if self.nb == 0:
            return []
        return self.root['objetsTouristiques']

    def next_row(self):
        while True:
            params = {
                'apiKey': self.api_key,
                'projetId': self.project_id,
                'selectionIds': [self.selection_id],
                'count': self.size,
                'first': self.skip,
                'responseFields': self.responseFields
            }
            response = self.request_or_retry(self.url, params={'query': json.dumps(params)})
            self.root = response.json()
            self.nb = int(self.root['numFound'])
            for row in self.items:
                yield row
            self.skip += self.size
            if self.skip >= self.nb:
                return

    def normalize_field_name(self, name):
        return name

    def filter_eid(self, src, val):
        return str(val)

    def filter_geom(self, src, val):
        lng, lat = val
        geom = Point(float(lng), float(lat), srid=4326)  # WGS84
        geom.transform(settings.SRID)
        return geom

    def _filter_comm(self, val, code, multiple=True):
        if not val:
            return None
        vals = [subval['coordonnees']['fr'] for subval in val if subval['type']['id'] == code]
        if multiple:
            return ' / '.join(vals)
        if vals:
            return vals[0]
        return None


class AttachmentApidaeParserMixin(object):
    def filter_attachments(self, src, val):
        result = []
        for subval in val or []:
            copyright_attachment = subval.get('copyright', {}).get('libelleFr')
            legend_attachment = subval.get('legende', {}).get('libelleFr')
            name_attachment = subval.get('nom', {}).get('libelleFr')
            if legend_attachment:
                legend = legend_attachment
                if guess_type(legend)[0] in ['image/jpeg', 'image/png', 'image/x-ms-bmp']:
                    legend = name_attachment
            else:
                legend = name_attachment
            result.append((subval['traductionFichiers'][0]['url'], legend, copyright_attachment))
        return result


class InformationDeskApidaeParser(ApidaeParser):
    type = None
    model = InformationDesk
    fields = {
        'eid': 'id',
        'description': 'presentation.descriptifDetaille.libelleFr',
        'geom': 'localisation.geolocalisation.geoJson.coordinates',
        'phone': 'informations.moyensCommunication',
        'email': 'informations.moyensCommunication',
        'website': 'informations.moyensCommunication',
        'photo': 'illustrations',
        'street': ('localisation.adresse.adresse1',
                   'localisation.adresse.adresse2',
                   'localisation.adresse.adresse3',),<commit_msg>Add docstrings<commit_after>import json
import os

import datetime

from mimetypes import guess_type
from urllib.parse import urlparse

from django.conf import settings
from django.contrib.gis.geos import Point
from django.core.files.base import ContentFile
from django.db import models
from django.utils.translation import gettext as _
from django.core.files.uploadedfile import UploadedFile

from geotrek.common.parsers import (AttachmentParserMixin, Parser,
                                    TourInSoftParser)
from geotrek.tourism.models import (InformationDesk, TouristicContent, TouristicEvent,
                                    TouristicContentType1, TouristicContentType2)


class TouristicContentMixin:
    # Mixin which handle multiple type1/2 with the same name in different categories
    def get_to_delete_kwargs(self):
        # FIXME: use mapping if it exists
        kwargs = {}
        for dst, val in self.constant_fields.items():
            field = self.model._meta.get_field(dst)
            if isinstance(field, models.ForeignKey):
                natural_key = self.natural_keys[dst]
                try:
                    kwargs[dst] = field.remote_field.model.objects.get(**{natural_key: val})
                except field.remote_field.model.DoesNotExist:
                    return None
            else:
                kwargs[dst] = val
        for dst, val in self.m2m_constant_fields.items():
            assert not self.separator or self.separator not in val
            field = self.model._meta.get_field(dst)
            natural_key = self.natural_keys[dst]
            filters = {natural_key: subval for subval in val}
            if not filters:
                continue
            if dst in ('type1', 'type2'):
                filters['category'] = kwargs['category'].pk
            try:
                kwargs[dst] = field.remote_field.model.objects.get(**filters)
            except field.remote_field.model.DoesNotExist:
                return None
        return kwargs


class ApidaeParser(AttachmentParserMixin, Parser):
    """Parser to import "anything" from APIDAE"""
    separator = None
    api_key = None
    project_id = None
    selection_id = None
    url = 'http://api.apidae-tourisme.com/api/v002/recherche/list-objets-touristiques/'
    model = None
    eid = 'eid'
    fields = {
        'name': 'nom.libelleFr',
    }
    constant_fields = {
        'published': True,
    }
    non_fields = {
        'attachments': 'illustrations',
    }
    # Use for foreign keys. When the key is a foreign key, it will try to get the key's value.
    # In django : 'category' : 'label' -> category.label
    field_options = {
        'name': {'required': True},
        'geom': {'required': True},
    }
    size = 100
    skip = 0
    responseFields = [
        'id',
        'nom',
        'presentation.descriptifCourt',
        'presentation.descriptifDetaille',
        'localisation.adresse',
        'localisation.geolocalisation.geoJson.coordinates',
        'localisation.geolocalisation.complement.libelleFr',
        'informations.moyensCommunication',
        'ouverture.periodeEnClair',
        'informationsHebergementCollectif.capacite.capaciteTotale',
        'informationsHebergementCollectif.hebergementCollectifType.libelleFr',
        'descriptionTarif.tarifsEnClair',
        'descriptionTarif.modesPaiement',
        'prestations.services',
        'gestion.dateModification',
        'gestion.membreProprietaire.nom',
        'illustrations'
    ]

    @property
    def items(self):
        if self.nb == 0:
            return []
        return self.root['objetsTouristiques']

    def next_row(self):
        while True:
            params = {
                'apiKey': self.api_key,
                'projetId': self.project_id,
                'selectionIds': [self.selection_id],
                'count': self.size,
                'first': self.skip,
                'responseFields': self.responseFields
            }
            response = self.request_or_retry(self.url, params={'query': json.dumps(params)})
            self.root = response.json()
            self.nb = int(self.root['numFound'])
            for row in self.items:
                yield row
            self.skip += self.size
            if self.skip >= self.nb:
                return

    def normalize_field_name(self, name):
        return name

    def filter_eid(self, src, val):
        return str(val)

    def filter_geom(self, src, val):
        lng, lat = val
        geom = Point(float(lng), float(lat), srid=4326)  # WGS84
        geom.transform(settings.SRID)
        return geom

    def _filter_comm(self, val, code, multiple=True):
        """
        With the code of the section (mail, phone etc...), allow to get the rigth information from 'informations.moyensCommunication'
        When the information is multiple (val is a list), you can either get the first element (multiple=False),
        either get all elements separated by a / (multiple=True)
        """
        if not val:
            return None
        vals = [subval['coordonnees']['fr'] for subval in val if subval['type']['id'] == code]
        if multiple:
            return ' / '.join(vals)
        if vals:
            return vals[0]
        return None


class AttachmentApidaeParserMixin(object):
    def filter_attachments(self, src, val):
        result = []
        for subval in val or []:
            copyright_attachment = subval.get('copyright', {}).get('libelleFr')
            legend_attachment = subval.get('legende', {}).get('libelleFr')
            name_attachment = subval.get('nom', {}).get('libelleFr')
            if legend_attachment:
                legend = legend_attachment
                if guess_type(legend)[0] in ['image/jpeg', 'image/png', 'image/x-ms-bmp']:
                    legend = name_attachment
            else:
                legend = name_attachment
            result.append((subval['traductionFichiers'][0]['url'], legend, copyright_attachment))
        return result


class InformationDeskApidaeParser(ApidaeParser):
    """Parser to import information desks from APIDAE"""
    type = None
    model = InformationDesk
    fields = {
        'eid': 'id',
        'description': 'presentation.descriptifDetaille.libelleFr',
        'geom': 'localisation.geolocalisation.geoJson.coordinates',
        'phone': 'informations.moyensCommunication',
        'email': 'informations.moyensCommunication',
        'website': 'informations.moyensCommunication',
        'photo': 'illustrations',
        'street': ('localisation.adresse.adresse1',
                   'localisation.adresse.adresse2',
                   'localisation.adresse.adresse3',),
{'fraction_non_alphanumeric': 0.10327668044709845, 'fraction_numerical': 0.004134129536058797, 'mean_word_length': 2.182216808769793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4980390', 'n_tokens_mistral': 3788, 'n_tokens_neox': 3655, 'n_words': 839}