<commit_before>"""
Usage:

    import_localities < Localities.csv
"""
from ciso8601 import parse_datetime
from django.utils.timezone import make_aware
from django.contrib.gis.geos import GEOSGeometry
from django.utils.text import slugify
from ..import_from_csv import ImportFromCSVCommand
from ...models import Locality


class Command(ImportFromCSVCommand):
    """
    Imports localities from the NPTG
    """
    def handle_rows(self, rows):
        existing_localities = Locality.objects.in_bulk()
        slugs = {
            locality.slug: locality for locality in existing_localities.values()
        }
        to_update = []
        to_create = []

        for row in rows:
            locality_code = row['NptgLocalityCode']
            if locality_code in existing_localities:
                locality = existing_localities[locality_code]
            else:
                locality = Locality()

            modified_at = row['ModificationDateTime']
            if modified_at:
                modified_at = parse_datetime(modified_at)
                if not modified_at.tzinfo:
                    modified_at = make_aware(modified_at)

            created_at = parse_datetime(row["CreationDateTime"])
            if not created_at.tzinfo:
                created_at = make_aware(created_at)

            if locality.id and modified_at == locality.modified_at:
                continue

            locality.modified_at = modified_at
            locality.created_at = created_at

            locality.name = row['LocalityName'].replace('\'', '\u2019')
            locality.short_name = row['ShortName']
            if locality.name == locality.short_name:
                locality.short_name = ''
            locality.qualifier_name = row['QualifierName']
            locality.admin_area_id = row['AdministrativeAreaCode']
            locality.latlong = GEOSGeometry(f"SRID=27700;POINT({row['Easting']} {row['Northing']})")

            if row['NptgDistrictCode'] == '310':  # bogus code seemingly used for localities with no district<commit_msg>slightly speed up localities import<commit_after>"""
Usage:

    import_localities < Localities.csv
"""
from ciso8601 import parse_datetime
from django.utils.timezone import make_aware
from django.contrib.gis.geos import GEOSGeometry
from django.utils.text import slugify
from ..import_from_csv import ImportFromCSVCommand
from ...models import Locality


class Command(ImportFromCSVCommand):
    """
    Imports localities from the NPTG
    """
    def handle_rows(self, rows):
        existing_localities = Locality.objects.defer('search_vector', 'latlong').in_bulk()
        slugs = {
            locality.slug: locality for locality in existing_localities.values()
        }
        to_update = []
        to_create = []

        for row in rows:
            modified_at = row['ModificationDateTime']
            modified_at = parse_datetime(modified_at)
            if not modified_at.tzinfo:
                modified_at = make_aware(modified_at)

            locality_code = row['NptgLocalityCode']
            if locality_code in existing_localities:
                locality = existing_localities[locality_code]
                if modified_at and modified_at == locality.modified_at:
                    continue
            else:
                locality = Locality()

            created_at = parse_datetime(row["CreationDateTime"])
            if not created_at.tzinfo:
                created_at = make_aware(created_at)

            locality.modified_at = modified_at
            locality.created_at = created_at

            locality.name = row['LocalityName'].replace('\'', '\u2019')
            locality.short_name = row['ShortName']
            if locality.name == locality.short_name:
                locality.short_name = ''
            locality.qualifier_name = row['QualifierName']
            locality.admin_area_id = row['AdministrativeAreaCode']
            locality.latlong = GEOSGeometry(f"SRID=27700;POINT({row['Easting']} {row['Northing']})")

            if row['NptgDistrictCode'] == '310':  # bogus code seemingly used for localities with no district
{'fraction_non_alphanumeric': 0.08891036482242087, 'fraction_numerical': 0.007731336071514858, 'mean_word_length': 2.413025556471558, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7652985', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1106, 'n_words': 242}