<commit_before>import warnings
import re

from django.contrib.gis.geos import Point
from django.contrib.gis.measure import D
from django.http import Http404
from django.db.models import Count
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.shortcuts import redirect
from django.core.urlresolvers import reverse
from django.views.generic import RedirectView, TemplateView
from django.shortcuts import get_object_or_404
from django import forms
from django.utils.translation import ugettext_lazy as _

import mapit
from haystack.views import SearchView
from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery
from haystack.forms import SearchForm

from popit.models import Person as PopitPerson
from speeches.models import Section, Speech, Speaker, Tag
from speeches.views import NamespaceMixin

from pombola.core import models
from pombola.core.views import PlaceDetailView, PlaceDetailSub, \
    OrganisationDetailView, PersonDetail, PlaceDetailView
from pombola.info.views import InfoPageView

from pombola.south_africa.models import ZAPlace

# In the short term, until we have a list of constituency offices and
# addresses from DA, let's bundle these together.
CONSTITUENCY_OFFICE_PLACE_KIND_SLUGS = (
    'constituency-office',
    'constituency-area', # specific to DA party
)

class LocationSearchForm(SearchForm):
    q = forms.CharField(required=False, label=_('Search'), widget=forms.TextInput(attrs={'placeholder': 'Your location'}))

class LatLonDetailBaseView(PlaceDetailView):

    # Using 25km as the default, as that's what's used on MyReps.
    constituency_office_search_radius = 25

    # The codes used here should match the party slugs, and the names of the
    # icon files in .../static/images/party-map-icons/
    party_slugs_that_have_logos = set((
        'adcp', 'anc', 'apc', 'azapo', 'cope', 'da', 'ff', 'id', 'ifp', 'mf',
        'pac', 'sacp', 'ucdp', 'udm'
    ));

    def get_object(self):
        # FIXME - handle bad args better.
        lat = float(self.kwargs['lat'])
        lon = float(self.kwargs['lon'])

        self.location = Point(lon, lat)

        areas = mapit.models.Area.objects.by_location(self.location)

        try:
            # FIXME - Handle getting more than one province.
            province = models.Place.objects.get(mapit_area__in=areas, kind__slug='province')
        except models.Place.DoesNotExist:
            raise Http404

        return province

    def get_context_data(self, **kwargs):
        context = super(LatLonDetailBaseView, self).get_context_data(**kwargs)
        context['location'] = self.location

        context['office_search_radius'] = self.constituency_office_search_radius

        context['nearest_offices'] = nearest_offices = (
            ZAPlace.objects
            .filter(kind__slug__in=CONSTITUENCY_OFFICE_PLACE_KIND_SLUGS)
            .distance(self.location)
            .filter(location__distance_lte=(self.location, D(km=self.constituency_office_search_radius)))
            .order_by('distance')
            )

        national_assembly = models.Organisation.objects.get(slug="national-assembly")

        # FIXME - There must be a cleaner way/place to do this.
        for office in nearest_offices:
            try:
                cc_positions = office \
                    .organisation \
                    .position_set \
                    .filter(
                        person__position__title__slug='constituency-contact',
                    )

                constituency_contacts = models.Person.objects.filter(position__in=cc_positions)
                office_people_entries = []

                for constituency_contact in constituency_contacts:
                    office_people_entries.append({
                        'person': constituency_contact,
                        'positions': constituency_contact.position_set.filter(organisation__in=[national_assembly, office.organisation]),
                    })

                if len(office_people_entries):
                    office.office_people_entries = office_people_entries

            except models.Person.DoesNotExist:
                warnings.warn("{0} has no MPs".format(office.organisation))

            # Try to extract the political membership of this person and store
            # it next to the office. TODO - deal with several parties sharing
            # an office (should this happen?) and MPs with no party connection.
            if hasattr(office, 'office_people_entries'):
                for entry in office.office_people_entries:
                    try:
                        party_slug = entry['person'].position_set.filter(title__slug="member", organisation__kind__slug="party")[0].organisation.slug
                        if party_slug in self.party_slugs_that_have_logos:
                            office.party_slug_for_icon = party_slug
                    except IndexError:
                        warnings.warn("{0} has no party membership".format(entry['person']))
<commit_msg>Tidy up query and add comment<commit_after>import warnings
import re

from django.contrib.gis.geos import Point
from django.contrib.gis.measure import D
from django.http import Http404
from django.db.models import Count
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.shortcuts import redirect
from django.core.urlresolvers import reverse
from django.views.generic import RedirectView, TemplateView
from django.shortcuts import get_object_or_404
from django import forms
from django.utils.translation import ugettext_lazy as _

import mapit
from haystack.views import SearchView
from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery
from haystack.forms import SearchForm

from popit.models import Person as PopitPerson
from speeches.models import Section, Speech, Speaker, Tag
from speeches.views import NamespaceMixin

from pombola.core import models
from pombola.core.views import PlaceDetailView, PlaceDetailSub, \
    OrganisationDetailView, PersonDetail, PlaceDetailView
from pombola.info.views import InfoPageView

from pombola.south_africa.models import ZAPlace

# In the short term, until we have a list of constituency offices and
# addresses from DA, let's bundle these together.
CONSTITUENCY_OFFICE_PLACE_KIND_SLUGS = (
    'constituency-office',
    'constituency-area', # specific to DA party
)

class LocationSearchForm(SearchForm):
    q = forms.CharField(required=False, label=_('Search'), widget=forms.TextInput(attrs={'placeholder': 'Your location'}))

class LatLonDetailBaseView(PlaceDetailView):

    # Using 25km as the default, as that's what's used on MyReps.
    constituency_office_search_radius = 25

    # The codes used here should match the party slugs, and the names of the
    # icon files in .../static/images/party-map-icons/
    party_slugs_that_have_logos = set((
        'adcp', 'anc', 'apc', 'azapo', 'cope', 'da', 'ff', 'id', 'ifp', 'mf',
        'pac', 'sacp', 'ucdp', 'udm'
    ));

    def get_object(self):
        # FIXME - handle bad args better.
        lat = float(self.kwargs['lat'])
        lon = float(self.kwargs['lon'])

        self.location = Point(lon, lat)

        areas = mapit.models.Area.objects.by_location(self.location)

        try:
            # FIXME - Handle getting more than one province.
            province = models.Place.objects.get(mapit_area__in=areas, kind__slug='province')
        except models.Place.DoesNotExist:
            raise Http404

        return province

    def get_context_data(self, **kwargs):
        context = super(LatLonDetailBaseView, self).get_context_data(**kwargs)
        context['location'] = self.location

        context['office_search_radius'] = self.constituency_office_search_radius

        context['nearest_offices'] = nearest_offices = (
            ZAPlace.objects
            .filter(kind__slug__in=CONSTITUENCY_OFFICE_PLACE_KIND_SLUGS)
            .distance(self.location)
            .filter(location__distance_lte=(self.location, D(km=self.constituency_office_search_radius)))
            .order_by('distance')
            )

        # FIXME - There must be a cleaner way/place to do this.
        for office in nearest_offices:
            try:
                cc_positions = office \
                    .organisation \
                    .position_set \
                    .filter(
                        person__position__title__slug='constituency-contact',
                    )

                constituency_contacts = models.Person.objects.filter(position__in=cc_positions)
                office_people_entries = []

                for constituency_contact in constituency_contacts:

                    # Find positions for this person that are relevant to the office.
                    positions = constituency_contact.position_set \
                        .filter(
                            organisation__slug__in = [
                                "national-assembly",
                                office.organisation.slug,
                            ]
                        )

                    office_people_entries.append({
                        'person': constituency_contact,
                        'positions': positions
                    })

                if len(office_people_entries):
                    office.office_people_entries = office_people_entries

            except models.Person.DoesNotExist:
                warnings.warn("{0} has no MPs".format(office.organisation))

            # Try to extract the political membership of this person and store
            # it next to the office. TODO - deal with several parties sharing
            # an office (should this happen?) and MPs with no party connection.
            if hasattr(office, 'office_people_entries'):
                for entry in office.office_people_entries:
                    try:
                        party_slug = entry['person'].position_set.filter(title__slug="member", organisation__kind__slug="party")[0].organisation.slug
                        if party_slug in self.party_slugs_that_have_logos:
                            office.party_slug_for_icon = party_slug
                    except IndexError:
                        warnings.warn("{0} has no party membership".format(entry['person']))

{'fraction_non_alphanumeric': 0.07903086241707528, 'fraction_numerical': 0.0030766272473800598, 'mean_word_length': 2.513002364066194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4306721', 'n_tokens_mistral': 2852, 'n_tokens_neox': 2731, 'n_words': 713}