<commit_before>import re
import uuid

from django.conf import settings
from django.contrib.admin.widgets import AdminTextareaWidget
from django.contrib.gis.db import models
from django.core.validators import RegexValidator, FileExtensionValidator
from django.db.models import JSONField
from django.utils import timezone
from django.utils.html import mark_safe, format_html, format_html_join
from django.utils.translation import gettext_lazy as _
from django_countries.fields import CountryField
from dynamic_filenames import FilePattern
from stdimage.models import StdImageField

from agir.lib import data
from agir.lib.data import departement_from_zipcode, FRANCE_COUNTRY_CODES
from agir.lib.utils import resize_and_autorotate
from .display import display_address
from .form_fields import RichEditorWidget, AdminRichEditorWidget
from .html import sanitize_html

RE_FRENCH_ZIPCODE = re.compile("^[0-9]{5}$")


class TimeStampedModel(models.Model):
    created = models.DateTimeField(
        "date de création", default=timezone.now, editable=False
    )
    modified = models.DateTimeField("dernière modification", auto_now=True)

    class Meta:
        abstract = True


class UUIDIdentified(models.Model):
    """
    Mixin that replaces the default id by an UUID
    """

    id = models.UUIDField(
        _("UUID"),
        primary_key=True,
        default=uuid.uuid4,
        editable=False,
        help_text=_("UUID interne à l'API pour identifier la ressource"),
    )

    class Meta:
        abstract = True


class BaseAPIResource(UUIDIdentified, TimeStampedModel):
    """
    Abstract base class for APIResource

    Automatically add an UUID identifier, and automatic
    timestamps on modification and creation
    """

    class Meta:
        abstract = True


class LocationMixin(models.Model):
    """
    Mixin that adds location fields
    """

    COORDINATES_MANUAL = 0
    COORDINATES_EXACT = 10
    COORDINATES_STREET = 20
    COORDINATES_DISTRICT = 25
    COORDINATES_CITY = 30
    COORDINATES_UNKNOWN_PRECISION = 50
    COORDINATES_NO_POSITION = 254
    COORDINATES_NOT_FOUND = 255
    COORDINATES_TYPE_CHOICES = (
        (COORDINATES_MANUAL, _("Coordonnées manuelles")),
        (COORDINATES_EXACT, _("Coordonnées automatiques précises")),
        (COORDINATES_STREET, _("Coordonnées automatiques approximatives (niveau rue)")),
        (
            COORDINATES_DISTRICT,
            "Coordonnées automatique approximatives (arrondissement)",
        ),
        (COORDINATES_CITY, _("Coordonnées automatiques approximatives (ville)")),
        (
            COORDINATES_UNKNOWN_PRECISION,
            _("Coordonnées automatiques (qualité inconnue)"),
        ),
        (COORDINATES_NO_POSITION, _("Pas de position géographique")),
        (COORDINATES_NOT_FOUND, _("Coordonnées introuvables")),
    )

    GEOCODING_FIELDS = {
        "location_address1",
        "location_address2",
        "location_city",
        "location_zip",
        "location_state",
        "location_country",
    }

    coordinates = models.PointField(
        _("coordonnées"), geography=True, null=True, blank=True, spatial_index=True
    )
    coordinates_type = models.PositiveSmallIntegerField(
        _("type de coordonnées"),
        choices=COORDINATES_TYPE_CHOICES,
        null=True,
        editable=False,
        help_text=_("Comment les coordonnées ci-dessus ont-elle été acquises"),
    )

    location_name = models.CharField(_("nom du lieu"), max_length=255, blank=True)
    location_address1 = models.CharField(
        _("adresse (1ère ligne)"), max_length=100, blank=True
    )
    location_address2 = models.CharField(
        _("adresse (2ème ligne)"), max_length=100, blank=True
    )
    location_citycode = models.CharField(_("code INSEE"), max_length=20, blank=True)
    location_city = models.CharField(_("ville"), max_length=100, blank=True)
    location_zip = models.CharField(_("code postal"), max_length=20, blank=True)
    location_state = models.CharField(_("état"), max_length=40, blank=True)
    location_country = CountryField(
        _("pays"), blank=True, blank_label=_("(sélectionner un pays)"), default="FR"
    )

    def html_full_address(self):
        return display_address(self)

    def short_location(self):
        local_part = self.location_name
        if self.location_country == "FR":
            city_part = self.location_city
        elif self.location_city:
            city_part = self.location_city + ", " + self.location_country.name
        elif self.location_country:
            city_part = self.location_country.name
        else:
            city_part = ""

        if local_part and city_part:
            return "{} ({})".format(local_part, city_part)
        else:
            return local_part or city_part or _("Pas de localisation")

    @property
    def short_address(self):
        attrs = [
            "location_address1",
            "location_address2",
            "location_zip",
            "location_city",
        ]

        if self.location_country not in FRANCE_COUNTRY_CODES:
            attrs.extend(["location_state", "location_country"])

        return ", ".join(
            str(getattr(self, attr)) for attr in attrs if getattr(self, attr)
        )

    def has_location(self):
        return self.coordinates is not None and self.coordinates_type is not None

    def has_manual_location(self):
        return self.coordinates_type == self.COORDINATES_MANUAL

    def has_automatic_location(self):
        return (
            self.coordinates_type is not None
            and self.COORDINATES_MANUAL
            < self.coordinates_type
            < self.COORDINATES_NOT_FOUND
        )

    def should_relocate_when_address_changed(self):
        return not self.has_location() or self.has_automatic_location()

    @property
    def departement(self):
        if self.location_country in FRANCE_COUNTRY_CODES and RE_FRENCH_ZIPCODE.match(
            self.location_zip
        ):
            departement = departement_from_zipcode(self.location_zip)
            if departement:
                return departement["nom"]
<commit_msg>fix: allow re-geocoding on address change if item has no coordinates<commit_after>import re
import uuid

from django.conf import settings
from django.contrib.admin.widgets import AdminTextareaWidget
from django.contrib.gis.db import models
from django.core.validators import RegexValidator, FileExtensionValidator
from django.db.models import JSONField
from django.utils import timezone
from django.utils.html import mark_safe, format_html, format_html_join
from django.utils.translation import gettext_lazy as _
from django_countries.fields import CountryField
from dynamic_filenames import FilePattern
from stdimage.models import StdImageField

from agir.lib import data
from agir.lib.data import departement_from_zipcode, FRANCE_COUNTRY_CODES
from agir.lib.utils import resize_and_autorotate
from .display import display_address
from .form_fields import RichEditorWidget, AdminRichEditorWidget
from .html import sanitize_html

RE_FRENCH_ZIPCODE = re.compile("^[0-9]{5}$")


class TimeStampedModel(models.Model):
    created = models.DateTimeField(
        "date de création", default=timezone.now, editable=False
    )
    modified = models.DateTimeField("dernière modification", auto_now=True)

    class Meta:
        abstract = True


class UUIDIdentified(models.Model):
    """
    Mixin that replaces the default id by an UUID
    """

    id = models.UUIDField(
        _("UUID"),
        primary_key=True,
        default=uuid.uuid4,
        editable=False,
        help_text=_("UUID interne à l'API pour identifier la ressource"),
    )

    class Meta:
        abstract = True


class BaseAPIResource(UUIDIdentified, TimeStampedModel):
    """
    Abstract base class for APIResource

    Automatically add an UUID identifier, and automatic
    timestamps on modification and creation
    """

    class Meta:
        abstract = True


class LocationMixin(models.Model):
    """
    Mixin that adds location fields
    """

    COORDINATES_MANUAL = 0
    COORDINATES_EXACT = 10
    COORDINATES_STREET = 20
    COORDINATES_DISTRICT = 25
    COORDINATES_CITY = 30
    COORDINATES_UNKNOWN_PRECISION = 50
    COORDINATES_NO_POSITION = 254
    COORDINATES_NOT_FOUND = 255
    COORDINATES_TYPE_CHOICES = (
        (COORDINATES_MANUAL, _("Coordonnées manuelles")),
        (COORDINATES_EXACT, _("Coordonnées automatiques précises")),
        (COORDINATES_STREET, _("Coordonnées automatiques approximatives (niveau rue)")),
        (
            COORDINATES_DISTRICT,
            "Coordonnées automatique approximatives (arrondissement)",
        ),
        (COORDINATES_CITY, _("Coordonnées automatiques approximatives (ville)")),
        (
            COORDINATES_UNKNOWN_PRECISION,
            _("Coordonnées automatiques (qualité inconnue)"),
        ),
        (COORDINATES_NO_POSITION, _("Pas de position géographique")),
        (COORDINATES_NOT_FOUND, _("Coordonnées introuvables")),
    )

    GEOCODING_FIELDS = {
        "location_address1",
        "location_address2",
        "location_city",
        "location_zip",
        "location_state",
        "location_country",
    }

    coordinates = models.PointField(
        _("coordonnées"), geography=True, null=True, blank=True, spatial_index=True
    )
    coordinates_type = models.PositiveSmallIntegerField(
        _("type de coordonnées"),
        choices=COORDINATES_TYPE_CHOICES,
        null=True,
        editable=False,
        help_text=_("Comment les coordonnées ci-dessus ont-elle été acquises"),
    )

    location_name = models.CharField(_("nom du lieu"), max_length=255, blank=True)
    location_address1 = models.CharField(
        _("adresse (1ère ligne)"), max_length=100, blank=True
    )
    location_address2 = models.CharField(
        _("adresse (2ème ligne)"), max_length=100, blank=True
    )
    location_citycode = models.CharField(_("code INSEE"), max_length=20, blank=True)
    location_city = models.CharField(_("ville"), max_length=100, blank=True)
    location_zip = models.CharField(_("code postal"), max_length=20, blank=True)
    location_state = models.CharField(_("état"), max_length=40, blank=True)
    location_country = CountryField(
        _("pays"), blank=True, blank_label=_("(sélectionner un pays)"), default="FR"
    )

    def html_full_address(self):
        return display_address(self)

    def short_location(self):
        local_part = self.location_name
        if self.location_country == "FR":
            city_part = self.location_city
        elif self.location_city:
            city_part = self.location_city + ", " + self.location_country.name
        elif self.location_country:
            city_part = self.location_country.name
        else:
            city_part = ""

        if local_part and city_part:
            return "{} ({})".format(local_part, city_part)
        else:
            return local_part or city_part or _("Pas de localisation")

    @property
    def short_address(self):
        attrs = [
            "location_address1",
            "location_address2",
            "location_zip",
            "location_city",
        ]

        if self.location_country not in FRANCE_COUNTRY_CODES:
            attrs.extend(["location_state", "location_country"])

        return ", ".join(
            str(getattr(self, attr)) for attr in attrs if getattr(self, attr)
        )

    def has_location(self):
        return self.coordinates is not None and self.coordinates_type is not None

    def has_manual_location(self):
        return self.coordinates_type == self.COORDINATES_MANUAL

    def has_automatic_location(self):
        return (
            self.coordinates_type is not None
            and self.COORDINATES_MANUAL
            < self.coordinates_type
            < self.COORDINATES_NOT_FOUND
        )

    def should_relocate_when_address_changed(self):
        return not self.has_location() or not self.has_manual_location()

    @property
    def departement(self):
        if self.location_country in FRANCE_COUNTRY_CODES and RE_FRENCH_ZIPCODE.match(
            self.location_zip
        ):
            departement = departement_from_zipcode(self.location_zip)
            if departement:
                return departement["nom"]

{'fraction_non_alphanumeric': 0.08670988654781199, 'fraction_numerical': 0.007617504051863858, 'mean_word_length': 3.018560729404103, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2178750', 'n_tokens_mistral': 3813, 'n_tokens_neox': 3686, 'n_words': 815}