<commit_before>from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.gis.db import models
from django.contrib.gis.geos import Point
from django.core.validators import MinValueValidator
from django.core.urlresolvers import reverse
from django.dispatch import receiver
from django.db import IntegrityError
from django.db.models import Q
from django.forms.models import model_to_dict
from django.utils import dateparse

import json
import pickle
from urllib import request
import utils

import jsonschema
import pytz


class ProposalManager(models.GeoManager):
    def latest(self):
        results = self.order_by("-created")
        return results and results[0]

    def between(self, start=None, end=None):
        q = None

        if start:
            q = Q(created__gte=start)

        if end:
            endQ = Q(closed_lte=end)

            if q:
                q = Q & endQ
            else:
                q = endQ

        return self.filter(q)

    def for_parcel(self, parcel):
        return self.filter(location__within=parcel.shape)


def make_property_map():
    def _g(p, default=""):
        return lambda d: d.get(p, default)

    def _G(p):
        return lambda d: d[p]

    def get_other_addresses(d):
        return ";".join(d["all_addresses"][1:]) if "all_addresses" in d else ""


    return [("address", lambda d: d["all_addresses"][0]),
            ("other_addresses", get_other_addresses),
            ("location", lambda d: Point(d["location"]["long"], d["location"]["lat"])),
            ("summary", lambda d: d.get("summary", "")[0:1024]),
            ("description", _g("description")),
            ("source", _g("source")),
            ("region_name", _g("region_name")),
            ("updated", utils.make_fn_chain(_G("updated_date"), dateparse.parse_datetime)),
            ("complete", _G("complete")),
            ("status", _g("status"))]


property_map = make_property_map()


class Proposal(models.Model):
    case_number = models.CharField(
        max_length=64,
        unique=True,
        help_text=("The unique case number "
                   "assigned by the city"))
    address = models.CharField(max_length=128, help_text="Street address")
    other_addresses = models.CharField(
        max_length=250,
        blank=True,
        help_text="Other addresses covered by this proposal")
    location = models.PointField(help_text="The latitude and longitude")
    region_name = models.CharField(
        max_length=128, default="Somerville, MA", null=True, help_text="")
    # The time when the proposal was last saved:
    modified = models.DateTimeField(auto_now=True)
    # The last time that the source was changed:
    updated = models.DateTimeField()
    created = models.DateTimeField(auto_now_add=True)
    summary = models.CharField(max_length=1024, default="")
    description = models.TextField(default="")
    source = models.URLField(
        null=True, help_text="The data source for the proposal.")
    status = models.CharField(max_length=64)

    # A proposal can be associated with a Project:
    project = models.ForeignKey("project.Project", blank=True, null=True)
    # A misnomer; if True, indicates that the planning board has issued a
    # ruling (approval or disapproval). Does not actually reflect whether the
    # proposed changes are done.
    complete = models.BooleanField(default=False)
    importer = models.ForeignKey("proposal.Importer", blank=True,
                                 null=True, on_delete=models.SET_NULL)

    parcel = models.ForeignKey(
        "parcel.Parcel", related_name="proposals", null=True, on_delete=models.SET_NULL)

    objects = ProposalManager()

    @classmethod
    def create_or_update_from_dict(cls, p_dict: dict):
        """
        Constructs a Proposal from a dictionary.  If an existing proposal has a
        matching case number, update it from p_dict.

        :param cls:
        :param p_dict: dictionary describing a proposal"""

        proposal: Proposal
        try:
            proposal = cls.objects.get(case_number=p_dict["case_number"])
            created = False
        except cls.DoesNotExist:
            proposal = cls(case_number=p_dict["case_number"])
            created = True

        proposal.update_from_dict(p_dict)

        return (created, proposal)


    @property
    def attribute_dict(self):
        return dict(self.attributes.values_list("name", "text_value"))

    def get_absolute_url(self):
        return reverse("view-proposal", kwargs={"pk": self.pk})

    def document_for_field(self, field):
        return self.documents.filter(field=field)

    def update_from_dict(self, p_dict):
        changed = bool(self.pk)

        if changed:
            prop_changes = []

        for prop, fn in property_map:
            old_val = changed and getattr(self, prop)
            try:
                val = fn(p_dict)
                if changed:
                    prop_changes.append({
                        "name": prop,
                        "new": val,
                        "old": old_val
                    })
                setattr(self, prop, val)
            except KeyError as exc:
                # The getters should throw a KeyError whenever a required
                # property is missing.

                # Don't raise an error if the property was already set.
                if old_val:
                    continue
                raise Exception("Missing required property: %s\n Reason: %s" %
                                (prop, exc))

        self.save()

        # Add related events:
        for event in self.create_events(p_dict.get("events", [])):
            event.proposals.add(self)

        # Create associated documents:
        document_fields = ["decisions", "reports", "other"]
        doc_dicts = zip(document_fields, map(p_dict.get, document_fields))
        self.create_documents((field, val["links"])
                              for field, val in doc_dicts
                              if val and val.get("links"))

        if changed:
            attr_changes = []
        for attr_name, attr_val in p_dict.get("attributes", []):
            try:
                handle = utils.normalize(attr_name)
                attr = self.attributes.get(handle=handle)
                old_val = attr.text_value
            except Attribute.DoesNotExist:
                self.attributes.create(
                    name=attr_name,
                    handle=handle,
                    text_value=attr_val,
                    published=p_dict["updated_date"])
                old_val = None
            if changed:
                attr_changes.append({
                    "name": attr_name,
                    "old": old_val,
                    "new": attr_val
                })

        if changed:
            changeset = Changeset.from_changes(self, {
                "properties": [ch for ch in prop_changes if ch["old"] != ch["new"]],
                "attributes": [ch for ch in attr_changes if ch["old"] != ch["new"]]
            })
            changeset.save()

    def create_documents(self, docs):
        for field, links in docs:<commit_msg>Fixed a couple of boneheaded issues when updating a proposal from a dict<commit_after>from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.gis.db import models
from django.contrib.gis.geos import Point
from django.core.validators import MinValueValidator
from django.core.urlresolvers import reverse
from django.dispatch import receiver
from django.db import IntegrityError
from django.db.models import Q
from django.forms.models import model_to_dict
from django.utils import dateparse

import json
import pickle
from urllib import request
import utils

import jsonschema
import pytz


class ProposalManager(models.GeoManager):
    def latest(self):
        results = self.order_by("-created")
        return results and results[0]

    def between(self, start=None, end=None):
        q = None

        if start:
            q = Q(created__gte=start)

        if end:
            endQ = Q(closed_lte=end)

            if q:
                q = Q & endQ
            else:
                q = endQ

        return self.filter(q)

    def for_parcel(self, parcel):
        return self.filter(location__within=parcel.shape)


def make_property_map():
    def _g(p, default=""):
        return lambda d: d.get(p, default)

    def _G(p):
        return lambda d: d[p]

    def get_other_addresses(d):
        return ";".join(d["all_addresses"][1:]) if "all_addresses" in d else ""


    return [("address", lambda d: d["all_addresses"][0]),
            ("other_addresses", get_other_addresses),
            ("location", lambda d: Point(d["location"]["long"], d["location"]["lat"])),
            ("summary", lambda d: d.get("summary", "")[0:1024]),
            ("description", _g("description")),
            ("source", _g("source")),
            ("region_name", _g("region_name")),
            ("updated", utils.make_fn_chain(_G("updated_date"), dateparse.parse_datetime)),
            ("complete", _G("complete")),
            ("status", _g("status"))]


property_map = make_property_map()


class Proposal(models.Model):
    case_number = models.CharField(
        max_length=64,
        unique=True,
        help_text=("The unique case number "
                   "assigned by the city"))
    address = models.CharField(max_length=128, help_text="Street address")
    other_addresses = models.CharField(
        max_length=250,
        blank=True,
        help_text="Other addresses covered by this proposal")
    location = models.PointField(help_text="The latitude and longitude")
    region_name = models.CharField(
        max_length=128, default="Somerville, MA", null=True, help_text="")
    # The time when the proposal was last saved:
    modified = models.DateTimeField(auto_now=True)
    # The last time that the source was changed:
    updated = models.DateTimeField()
    created = models.DateTimeField(auto_now_add=True)
    summary = models.CharField(max_length=1024, default="")
    description = models.TextField(default="")
    source = models.URLField(
        null=True, help_text="The data source for the proposal.")
    status = models.CharField(max_length=64)

    # A proposal can be associated with a Project:
    project = models.ForeignKey("project.Project", blank=True, null=True)
    # A misnomer; if True, indicates that the planning board has issued a
    # ruling (approval or disapproval). Does not actually reflect whether the
    # proposed changes are done.
    complete = models.BooleanField(default=False)
    importer = models.ForeignKey("proposal.Importer", blank=True,
                                 null=True, on_delete=models.SET_NULL)

    parcel = models.ForeignKey(
        "parcel.Parcel", related_name="proposals", null=True, on_delete=models.SET_NULL)

    objects = ProposalManager()

    @classmethod
    def create_or_update_from_dict(cls, p_dict: dict):
        """
        Constructs a Proposal from a dictionary.  If an existing proposal has a
        matching case number, update it from p_dict.

        :param cls:
        :param p_dict: dictionary describing a proposal"""

        proposal: Proposal
        try:
            proposal = cls.objects.get(case_number=p_dict["case_number"])
            created = False
        except cls.DoesNotExist:
            proposal = cls(case_number=p_dict["case_number"])
            created = True

        proposal.update_from_dict(p_dict)

        return (created, proposal)


    @property
    def attribute_dict(self):
        return dict(self.attributes.values_list("name", "text_value"))

    def get_absolute_url(self):
        return reverse("view-proposal", kwargs={"pk": self.pk})

    def document_for_field(self, field):
        return self.documents.filter(field=field)

    def update_from_dict(self, p_dict):
        changed = bool(self.pk)

        if changed:
            prop_changes = []

        for prop, fn in property_map:
            old_val = changed and getattr(self, prop)
            try:
                val = fn(p_dict)
                if changed:
                    prop_changes.append({
                        "name": prop,
                        "new": val,
                        "old": old_val
                    })
                setattr(self, prop, val)
            except KeyError as exc:
                # The getters should throw a KeyError whenever a required
                # property is missing.

                # Don't raise an error if the property was already set.
                if old_val is not None:
                    continue
                raise Exception("Missing required property: %s\n Reason: %s" %
                                (prop, exc))

        self.save()

        # Add related events:
        for event in self.create_events(p_dict.get("events", [])):
            event.proposals.add(self)

        # Create associated documents:
        document_fields = ["decisions", "reports", "other"]
        doc_dicts = zip(document_fields, map(p_dict.get, document_fields))
        self.create_documents((field, val["links"])
                              for field, val in doc_dicts
                              if val and val.get("links"))

        if changed:
            attr_changes = []
        for attr_name, attr_val in p_dict.get("attributes", {}).items():
            try:
                handle = utils.normalize(attr_name)
                attr = self.attributes.get(handle=handle)
                old_val = attr.text_value
            except Attribute.DoesNotExist:
                self.attributes.create(
                    name=attr_name,
                    handle=handle,
                    text_value=attr_val,
                    published=p_dict["updated_date"])
                old_val = None
            if changed:
                attr_changes.append({
                    "name": attr_name,
                    "old": old_val,
                    "new": attr_val
                })

        if changed:
            changeset = Changeset.from_changes(self, {
                "properties": [ch for ch in prop_changes if ch["old"] != ch["new"]],
                "attributes": [ch for ch in attr_changes if ch["old"] != ch["new"]]
            })
            changeset.save()

    def create_documents(self, docs):
        for field, links in docs:
{'fraction_non_alphanumeric': 0.10213383053656515, 'fraction_numerical': 0.0034528002209792143, 'mean_word_length': 2.1298897773935597, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4791475', 'n_tokens_mistral': 4077, 'n_tokens_neox': 3914, 'n_words': 1051}