<commit_before>"""Command to import PMG's constitiency office data

PMG are undertaking an exercise to visit all the constituency
offices and collect accurate locations and photographs.
The data from this is available at

https://app.m4jam.com/app/campaigns/2298/export/

This script parses the resulting CSV file and updates our data
on constituency offices to have the correct location.
"""


import csv
import os.path
import re
from StringIO import StringIO

import requests

from django.conf import settings

from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.core.management.base import BaseCommand, CommandError
from django.contrib.gis.geos import Point

from pombola.core.models import (
    Organisation,
    )
from pombola.core.utils import mkdir_p

from pombola.images.models import Image


# Matches the format of the locations in the CSV we get from PMG.
location_re = re.compile(
    r'SRID=4326;POINT\((-?[0-9]+\.[0-9]+) (-?[0-9]+\.[0-9]+)\)')


class Command(BaseCommand):
    args = '<path to csv file>'
    help = 'Updates constituency offices based on the supplied CSV file.'

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError(
                "You must provide the path to the CSV file as an argument."
                )

        constituency_offices = Organisation.objects.filter(
            kind__slug='constituency-office')

        storage = FileSystemStorage()

        storage_path = os.path.join('images', 'constituency-offices')
        images_directory = os.path.join(
            settings.MEDIA_ROOT,
            storage_path
            )
        mkdir_p(images_directory)

        with open(args[0]) as csvfile:
            reader = csv.DictReader(csvfile)

            for row in reader:
                poi_ref = row['POI_REFERENCE']
                party, name = (
                    re.match(
                        '(.*) Constituency Office (.*)', poi_ref
                        ).groups()
                    )
                name = re.sub('\s+', ' ', name)

                qs = constituency_offices.filter(
                    name__regex=r'^{}.+: {}$'.format(party, name))

                try:
                    org = qs.get()
                except Organisation.MultipleObjectsReturned:
                    print "Skipping {} as multiple orgs returned: {}".format(
                        poi_ref,
                        repr(qs),
                        )
                    continue
                except Organisation.DoesNotExist:
                    # Fall back to searching for the name and the party in the
                    # constituency office name
                    qs = (
                        Organisation.objects
                        .filter(kind__slug='constituency-office')
                        .filter(name__contains=name)
                        .filter(name__contains=party)
                        )

                    org = qs.get()

                place = (
                    org.place_set
                    .filter(name__contains='Approximate position of')
                    .filter(kind__slug='constituency-office')
                    .get()  # There should be only one.
                    )

                lon, lat = location_re.match(row['Constituency_Pin']).groups()

                place.location = Point(float(lon), float(lat))
                place.save()

                # Now get the photo
                photo_url = row.get('Constituency_Photo')

                if photo_url:
                    response = requests.get(photo_url)
                    image_filename = '{}'.format(org.slug)

                    desired_storage_path = os.path.join(storage_path, image_filename)
                    storage_filename = storage.save(desired_storage_path, StringIO(response.content))

                    image = Image(
                        content_object=org,
                        source=photo_url,
                        is_primary=True,
                        image=storage_filename,
                        )
                    image.save()
<commit_msg>Skip offices with an image with the same URL.<commit_after>"""Command to import PMG's constitiency office data

PMG are undertaking an exercise to visit all the constituency
offices and collect accurate locations and photographs.
The data from this is available at

https://app.m4jam.com/app/campaigns/2298/export/

This script parses the resulting CSV file and updates our data
on constituency offices to have the correct location.
"""


import csv
import os.path
import re
from StringIO import StringIO

import requests

from django.conf import settings

from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.core.management.base import BaseCommand, CommandError
from django.contrib.gis.geos import Point

from pombola.core.models import (
    Organisation,
    )
from pombola.core.utils import mkdir_p

from pombola.images.models import Image


# Matches the format of the locations in the CSV we get from PMG.
location_re = re.compile(
    r'SRID=4326;POINT\((-?[0-9]+\.[0-9]+) (-?[0-9]+\.[0-9]+)\)')


class Command(BaseCommand):
    args = '<path to csv file>'
    help = 'Updates constituency offices based on the supplied CSV file.'

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError(
                "You must provide the path to the CSV file as an argument."
                )

        constituency_offices = Organisation.objects.filter(
            kind__slug='constituency-office')

        storage = FileSystemStorage()

        storage_path = os.path.join('images', 'constituency-offices')
        images_directory = os.path.join(
            settings.MEDIA_ROOT,
            storage_path
            )
        mkdir_p(images_directory)

        with open(args[0]) as csvfile:
            reader = csv.DictReader(csvfile)

            for row in reader:
                poi_ref = row['POI_REFERENCE']
                party, name = (
                    re.match(
                        '(.*) Constituency Office (.*)', poi_ref
                        ).groups()
                    )
                name = re.sub('\s+', ' ', name)

                qs = constituency_offices.filter(
                    name__regex=r'^{}.+: {}$'.format(party, name))

                try:
                    org = qs.get()
                except Organisation.MultipleObjectsReturned:
                    print "Skipping {} as multiple orgs returned: {}".format(
                        poi_ref,
                        repr(qs),
                        )
                    continue
                except Organisation.DoesNotExist:
                    # Fall back to searching for the name and the party in the
                    # constituency office name
                    qs = (
                        Organisation.objects
                        .filter(kind__slug='constituency-office')
                        .filter(name__contains=name)
                        .filter(name__contains=party)
                        )

                    org = qs.get()

                place = (
                    org.place_set
                    .filter(name__contains='Approximate position of')
                    .filter(kind__slug='constituency-office')
                    .get()  # There should be only one.
                    )

                lon, lat = location_re.match(row['Constituency_Pin']).groups()

                place.location = Point(float(lon), float(lat))
                place.save()

                # Now get the photo
                photo_url = row.get('Constituency_Photo')

                try:
                    org.images.get(source=photo_url)
                    print (
                        "Skipping {} as url matches existing image."
                        .format(org.slug)
                        )
                    continue
                except Image.DoesNotExist:
                    print "Adding new image to {}.".format(org.slug)

                if photo_url:
                    response = requests.get(photo_url)
                    image_filename = '{}'.format(org.slug)

                    desired_storage_path = os.path.join(storage_path, image_filename)
                    storage_filename = storage.save(desired_storage_path, StringIO(response.content))

                    image = Image(
                        content_object=org,
                        source=photo_url,
                        is_primary=True,
                        image=storage_filename,
                        )
                    image.save()

{'fraction_non_alphanumeric': 0.085486624928856, 'fraction_numerical': 0.004325554923164485, 'mean_word_length': 1.4946053378762068, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '424820', 'n_tokens_mistral': 2328, 'n_tokens_neox': 2141, 'n_words': 575}