<commit_before>from collections import Counter
from dateutil.parser import parse as date_parse
import datetime as dt

from django.db import models
from django.db.models import Sum, Count, F
from django.utils.translation import get_language
from django.contrib.postgres.fields import HStoreField

from standardisers import standardise_name, StandardiserNoOutputException, standardise_urn, standardise_postcode


STATUS_CHOICES = (("created_not_sent", "Created but not sent"),
                  ("sent", "Sent"),
                  ("network_error", "Network error"),
                  ("receipt_success", "Processed successfully"),
                  ("receipt_failure", "Processing failed"))


COURT_LANGUAGE_CHOICES = (("en", "English"),
                          ("cy", "Welsh"))


INITIATION_TYPE_CHOICES = (("C", "Charge"),
                           ("J", "SJP"),
                           ("Q", "Requisition"),
                           ("O", "Other"),
                           ("R", "Remitted"),
                           ("S", "Summons"))

NOTICE_TYPES_CHOICES = (("both", "Both"),
                        ("sjp", "SJP"),
                        ("non-sjp", "Non-SJP"))


def get_totals(qs):
    totals = qs.aggregate(Sum('total_pleas'),
                          Sum('total_guilty'),
                          Sum('total_not_guilty'))

    return {
        'submissions': qs.count(),
        'pleas': totals['total_pleas__sum'] or 0,
        'guilty': totals['total_guilty__sum'] or 0,
        'not_guilty': totals['total_not_guilty__sum'] or 0
    }


class CourtEmailCountManager(models.Manager):
    def calculate_aggregates(self, start_date, days=7):
        """
        Calculate aggregate stats (submissions, total pleas,
        guilty pleas, not guilty pleas) over the specified date period.
        """
        start_datetime = start_date

        end_datetime = start_date + dt.timedelta(days)

        qs = self.filter(sent=True,
                         court__test_mode=False,
                         date_sent__gte=start_datetime,
                         date_sent__lt=end_datetime)

        totals = get_totals(qs)

        return totals

    def get_stats(self, start=None, end=None):
        """
        Return stats, which can be filtered by start date or end date
        """
        qs = self.filter(sent=True, court__test_mode=False)

        if start:
            qs = qs.filter(date_sent__gte=start)

        if end:
            qs = qs.filter(date_sent__lte=end)

        return get_totals(qs)

    def get_stats_by_hearing_date(self, days=None, start_date=None):
        """
        Return stats grouped by hearing date starting from today
        for the number of days specified by days.
        """

        if not start_date:
            start_date = dt.date(2012,01,01)

        results = CourtEmailCount.objects\
            .filter(sent=True,
                    hearing_date__gte=start_date,
                    court__test_mode=False)\
            .extra({'hearing_day': "date(hearing_date)"})\
            .values('hearing_day')\
            .order_by('hearing_day')\
            .annotate(pleas=Sum('total_pleas'),
                      guilty=Sum('total_guilty'),
                      not_guilty=Sum('total_not_guilty'),
                      submissions=Count('hearing_date'))

        if days:
            results = results[:days]

        return results

    def get_stats_by_court(self, start=None, end=None):
        """
        Return stats grouped by court
        """
        courts = Court.objects.filter(test_mode=False)

        stats = []

        for court in courts:

            qs = self.filter(sent=True,
                             court__id=court.id)

            if start:
                qs = qs.filter(date_sent__gte=start)

            if end:
                qs = qs.filter(date_sent__lte=end)

            data = {"court_name": court.court_name,
                    "region_code": court.region_code}

            data.update(get_totals(qs))

            stats.append(data)

        return stats

    def get_stats_days_from_hearing(self, limit=60):
        courts = Court.objects.filter(test_mode=False)

        counts = CourtEmailCount.objects.filter(court__in=courts).annotate(num_days=F('hearing_date') - F('date_sent')).values('num_days').order_by("-num_days")
        day_counts = Counter([x["num_days"].days for x in counts if x["num_days"].days < limit and x["num_days"].days > 0])

        for day_number in range(limit):
            if day_number not in day_counts.keys():
                day_counts[day_number] = 0

        return day_counts


class CourtEmailCount(models.Model):
    date_sent = models.DateTimeField(auto_now_add=True)
    court = models.ForeignKey("Court", related_name="court_email_counts")
    initiation_type = models.CharField(max_length=2, null=False, blank=False, default="Q",
                                       choices=INITIATION_TYPE_CHOICES)
    language = models.CharField(max_length=2, null=False, blank=False, default="en",
                                choices=COURT_LANGUAGE_CHOICES)

    total_pleas = models.IntegerField()
    total_guilty = models.IntegerField()
    total_not_guilty = models.IntegerField()
    hearing_date = models.DateTimeField()

    # special circumstances char counts
    sc_guilty_char_count = models.PositiveIntegerField(default=0)
    sc_not_guilty_char_count = models.PositiveIntegerField(default=0)

    sent = models.BooleanField(null=False, default=False)
    processed = models.BooleanField(null=False, default=False)

    objects = CourtEmailCountManager()

    def get_status_from_case(self, case_obj):
        self.sent = case_obj.sent
        self.processed = case_obj.processed

    def get_from_context(self, context, court):
        if "plea" not in context:
            return
        if "data" not in context["plea"]:
            return
        if "your_details" not in context:
            return
        if "case" not in context:
            return

        if self.total_pleas is None:
            self.total_pleas = 0

        if self.total_guilty is None:
            self.total_guilty = 0

        if self.total_not_guilty is None:
            self.total_not_guilty = 0

        self.court = court

        try:
            if isinstance(context["case"]["contact_deadline"], dt.date):
                date_part = context["case"]["contact_deadline"]
            else:
                date_part = date_parse(context["case"]["contact_deadline"])

            self.hearing_date = date_part
        except KeyError:
            return False

        self.initiation_type = "J" if context["notice_type"]["sjp"] else "Q"
        self.language = get_language().split("-")[0]

        for plea_data in context["plea"]["data"]:
            self.total_pleas += 1

            if plea_data["guilty"] == "guilty":
                self.total_guilty += 1

            if plea_data["guilty"] == "not_guilty":
                self.total_not_guilty += 1

        # extra anon information
        self.sc_guilty_char_count, self.sc_not_guilty_char_count = 0, 0

        for plea in context["plea"]["data"]:
            if plea["guilty"] == "guilty":
                self.sc_guilty_char_count += len(plea.get("guilty_extra", ""))
            else:
                self.sc_not_guilty_char_count += len(plea.get("not_guilty_extra", ""))

        return True


class CaseManager(models.Manager):
    class Meta:
        ordering = ["offence_seq_number"]

    def can_use_urn(self, urn, first_name, last_name):
        name = standardise_name(first_name, last_name)

        cases = self.filter(urn__iexact=urn, sent=True)

        for case in cases:
            if case.extra_data and case.extra_data.get("OrganisationName"):
                return False
            elif case.name == name:
                return False
        return True

    def get_case_for_urn(self, urn):
        """
        Can the user use this URN for a DX based submission?
        """

        cases = self.filter(urn__iexact=urn, sent=False, imported=True)

        if not cases or cases.count() > 1 or not cases[0].can_auth():
            return None

        return cases[0]


class Case(models.Model):
    """
    The main case model.

    User data is gpg encrypted and persisted to disc then
    transferred to an encrypted S3 account.
    """
    created = models.DateTimeField(auto_now_add=True, null=True, blank=True)

    urn = models.CharField(max_length=30, db_index=True)

    extra_data = HStoreField(null=True, blank=True)

    case_number = models.CharField(max_length=12, null=True, blank=True,
                                   help_text="as supplied by DX")

    name = models.CharField(max_length=250, null=True, blank=True)
    email = models.EmailField(null=True, blank=True)
    email_permission = models.BooleanField(default=False)

    date_of_hearing = models.DateField(null=True, blank=True)
    imported = models.BooleanField(default=False)

    ou_code = models.CharField(max_length=10, null=True, blank=True)
    initiation_type = models.CharField(max_length=2, null=False, blank=False, default="Q",
                                       choices=INITIATION_TYPE_CHOICES)
    language = models.CharField(max_length=2, null=False, blank=False, default="en",
                                choices=COURT_LANGUAGE_CHOICES)

    sent = models.BooleanField(null=False, default=False)
    processed = models.BooleanField(null=False, default=False)

    objects = CaseManager()

    completed_on = models.DateTimeField(
        blank=True, null=True,
        help_text="The date/time a user completes a submission.")

    def add_action(self, status, status_info):
        self.actions.create(status=status, status_info=status_info)

    def has_action(self, status):
        return self.actions.filter(status=status).exists()

    def get_actions(self, status):
        return self.actions.filter(status=status)

    def get_users_name(self):
        """
        Attempt to get the user's name
        """

        if self.name:
            return self.name

        if self.extra_data and "Forename1" in self.extra_data:
            return "{} {}".format(self.extra_data["Forename1"], self.extra_data["Surname"])

        return ""

    def can_auth(self):
        """
        Do we have the relevant data to authenticate the user?
        """

        return self.extra_data and "PostCode" in self.extra_data or "DOB" in self.extra_data

    def authenticate(self, num_charges, postcode, dob):

        postcode_match, dob_match = False, False

        assert postcode or dob, "Provide at least one value"

        if not self.can_auth():
            return False

        if postcode and "PostCode" in self.extra_data:
            inputted_postcode = standardise_postcode(postcode)
            stored_postcode = standardise_postcode(self.extra_data.get("PostCode"))

            postcode_match = inputted_postcode == stored_postcode

        if dob and "DOB" in self.extra_data:
            dob_match = dob == date_parse(self.extra_data["DOB"]).date()

        return self.offences.count() == num_charges and (postcode_match or dob_match)

    def auth_field(self):
        """
        Determine which field to use for auth
        """

        if self.extra_data:
            if "DOB" in self.extra_data:
                return "DOB"
            elif "PostCode" in self.extra_data:<commit_msg>fix situation when extra_data is None not {}<commit_after>from collections import Counter
from dateutil.parser import parse as date_parse
import datetime as dt

from django.db import models
from django.db.models import Sum, Count, F
from django.utils.translation import get_language
from django.contrib.postgres.fields import HStoreField

from standardisers import standardise_name, StandardiserNoOutputException, standardise_urn, standardise_postcode


STATUS_CHOICES = (("created_not_sent", "Created but not sent"),
                  ("sent", "Sent"),
                  ("network_error", "Network error"),
                  ("receipt_success", "Processed successfully"),
                  ("receipt_failure", "Processing failed"))


COURT_LANGUAGE_CHOICES = (("en", "English"),
                          ("cy", "Welsh"))


INITIATION_TYPE_CHOICES = (("C", "Charge"),
                           ("J", "SJP"),
                           ("Q", "Requisition"),
                           ("O", "Other"),
                           ("R", "Remitted"),
                           ("S", "Summons"))

NOTICE_TYPES_CHOICES = (("both", "Both"),
                        ("sjp", "SJP"),
                        ("non-sjp", "Non-SJP"))


def get_totals(qs):
    totals = qs.aggregate(Sum('total_pleas'),
                          Sum('total_guilty'),
                          Sum('total_not_guilty'))

    return {
        'submissions': qs.count(),
        'pleas': totals['total_pleas__sum'] or 0,
        'guilty': totals['total_guilty__sum'] or 0,
        'not_guilty': totals['total_not_guilty__sum'] or 0
    }


class CourtEmailCountManager(models.Manager):
    def calculate_aggregates(self, start_date, days=7):
        """
        Calculate aggregate stats (submissions, total pleas,
        guilty pleas, not guilty pleas) over the specified date period.
        """
        start_datetime = start_date

        end_datetime = start_date + dt.timedelta(days)

        qs = self.filter(sent=True,
                         court__test_mode=False,
                         date_sent__gte=start_datetime,
                         date_sent__lt=end_datetime)

        totals = get_totals(qs)

        return totals

    def get_stats(self, start=None, end=None):
        """
        Return stats, which can be filtered by start date or end date
        """
        qs = self.filter(sent=True, court__test_mode=False)

        if start:
            qs = qs.filter(date_sent__gte=start)

        if end:
            qs = qs.filter(date_sent__lte=end)

        return get_totals(qs)

    def get_stats_by_hearing_date(self, days=None, start_date=None):
        """
        Return stats grouped by hearing date starting from today
        for the number of days specified by days.
        """

        if not start_date:
            start_date = dt.date(2012,01,01)

        results = CourtEmailCount.objects\
            .filter(sent=True,
                    hearing_date__gte=start_date,
                    court__test_mode=False)\
            .extra({'hearing_day': "date(hearing_date)"})\
            .values('hearing_day')\
            .order_by('hearing_day')\
            .annotate(pleas=Sum('total_pleas'),
                      guilty=Sum('total_guilty'),
                      not_guilty=Sum('total_not_guilty'),
                      submissions=Count('hearing_date'))

        if days:
            results = results[:days]

        return results

    def get_stats_by_court(self, start=None, end=None):
        """
        Return stats grouped by court
        """
        courts = Court.objects.filter(test_mode=False)

        stats = []

        for court in courts:

            qs = self.filter(sent=True,
                             court__id=court.id)

            if start:
                qs = qs.filter(date_sent__gte=start)

            if end:
                qs = qs.filter(date_sent__lte=end)

            data = {"court_name": court.court_name,
                    "region_code": court.region_code}

            data.update(get_totals(qs))

            stats.append(data)

        return stats

    def get_stats_days_from_hearing(self, limit=60):
        courts = Court.objects.filter(test_mode=False)

        counts = CourtEmailCount.objects.filter(court__in=courts).annotate(num_days=F('hearing_date') - F('date_sent')).values('num_days').order_by("-num_days")
        day_counts = Counter([x["num_days"].days for x in counts if x["num_days"].days < limit and x["num_days"].days > 0])

        for day_number in range(limit):
            if day_number not in day_counts.keys():
                day_counts[day_number] = 0

        return day_counts


class CourtEmailCount(models.Model):
    date_sent = models.DateTimeField(auto_now_add=True)
    court = models.ForeignKey("Court", related_name="court_email_counts")
    initiation_type = models.CharField(max_length=2, null=False, blank=False, default="Q",
                                       choices=INITIATION_TYPE_CHOICES)
    language = models.CharField(max_length=2, null=False, blank=False, default="en",
                                choices=COURT_LANGUAGE_CHOICES)

    total_pleas = models.IntegerField()
    total_guilty = models.IntegerField()
    total_not_guilty = models.IntegerField()
    hearing_date = models.DateTimeField()

    # special circumstances char counts
    sc_guilty_char_count = models.PositiveIntegerField(default=0)
    sc_not_guilty_char_count = models.PositiveIntegerField(default=0)

    sent = models.BooleanField(null=False, default=False)
    processed = models.BooleanField(null=False, default=False)

    objects = CourtEmailCountManager()

    def get_status_from_case(self, case_obj):
        self.sent = case_obj.sent
        self.processed = case_obj.processed

    def get_from_context(self, context, court):
        if "plea" not in context:
            return
        if "data" not in context["plea"]:
            return
        if "your_details" not in context:
            return
        if "case" not in context:
            return

        if self.total_pleas is None:
            self.total_pleas = 0

        if self.total_guilty is None:
            self.total_guilty = 0

        if self.total_not_guilty is None:
            self.total_not_guilty = 0

        self.court = court

        try:
            if isinstance(context["case"]["contact_deadline"], dt.date):
                date_part = context["case"]["contact_deadline"]
            else:
                date_part = date_parse(context["case"]["contact_deadline"])

            self.hearing_date = date_part
        except KeyError:
            return False

        self.initiation_type = "J" if context["notice_type"]["sjp"] else "Q"
        self.language = get_language().split("-")[0]

        for plea_data in context["plea"]["data"]:
            self.total_pleas += 1

            if plea_data["guilty"] == "guilty":
                self.total_guilty += 1

            if plea_data["guilty"] == "not_guilty":
                self.total_not_guilty += 1

        # extra anon information
        self.sc_guilty_char_count, self.sc_not_guilty_char_count = 0, 0

        for plea in context["plea"]["data"]:
            if plea["guilty"] == "guilty":
                self.sc_guilty_char_count += len(plea.get("guilty_extra", ""))
            else:
                self.sc_not_guilty_char_count += len(plea.get("not_guilty_extra", ""))

        return True


class CaseManager(models.Manager):
    class Meta:
        ordering = ["offence_seq_number"]

    def can_use_urn(self, urn, first_name, last_name):
        name = standardise_name(first_name, last_name)

        cases = self.filter(urn__iexact=urn, sent=True)

        for case in cases:
            if case.extra_data and case.extra_data.get("OrganisationName"):
                return False
            elif case.name == name:
                return False
        return True

    def get_case_for_urn(self, urn):
        """
        Can the user use this URN for a DX based submission?
        """

        cases = self.filter(urn__iexact=urn, sent=False, imported=True)

        if not cases or cases.count() > 1 or not cases[0].can_auth():
            return None

        return cases[0]


class Case(models.Model):
    """
    The main case model.

    User data is gpg encrypted and persisted to disc then
    transferred to an encrypted S3 account.
    """
    created = models.DateTimeField(auto_now_add=True, null=True, blank=True)

    urn = models.CharField(max_length=30, db_index=True)

    extra_data = HStoreField(null=True, blank=True)

    case_number = models.CharField(max_length=12, null=True, blank=True,
                                   help_text="as supplied by DX")

    name = models.CharField(max_length=250, null=True, blank=True)
    email = models.EmailField(null=True, blank=True)
    email_permission = models.BooleanField(default=False)

    date_of_hearing = models.DateField(null=True, blank=True)
    imported = models.BooleanField(default=False)

    ou_code = models.CharField(max_length=10, null=True, blank=True)
    initiation_type = models.CharField(max_length=2, null=False, blank=False, default="Q",
                                       choices=INITIATION_TYPE_CHOICES)
    language = models.CharField(max_length=2, null=False, blank=False, default="en",
                                choices=COURT_LANGUAGE_CHOICES)

    sent = models.BooleanField(null=False, default=False)
    processed = models.BooleanField(null=False, default=False)

    objects = CaseManager()

    completed_on = models.DateTimeField(
        blank=True, null=True,
        help_text="The date/time a user completes a submission.")

    def add_action(self, status, status_info):
        self.actions.create(status=status, status_info=status_info)

    def has_action(self, status):
        return self.actions.filter(status=status).exists()

    def get_actions(self, status):
        return self.actions.filter(status=status)

    def get_users_name(self):
        """
        Attempt to get the user's name
        """

        if self.name:
            return self.name

        if self.extra_data and "Forename1" in self.extra_data:
            return "{} {}".format(self.extra_data["Forename1"], self.extra_data["Surname"])

        return ""

    def can_auth(self):
        """
        Do we have the relevant data to authenticate the user?
        """

        return self.extra_data and ("PostCode" in self.extra_data or "DOB" in self.extra_data)

    def authenticate(self, num_charges, postcode, dob):

        postcode_match, dob_match = False, False

        assert postcode or dob, "Provide at least one value"

        if not self.can_auth():
            return False

        if postcode and "PostCode" in self.extra_data:
            inputted_postcode = standardise_postcode(postcode)
            stored_postcode = standardise_postcode(self.extra_data.get("PostCode"))

            postcode_match = inputted_postcode == stored_postcode

        if dob and "DOB" in self.extra_data:
            dob_match = dob == date_parse(self.extra_data["DOB"]).date()

        return self.offences.count() == num_charges and (postcode_match or dob_match)

    def auth_field(self):
        """
        Determine which field to use for auth
        """

        if self.extra_data:
            if "DOB" in self.extra_data:
                return "DOB"
            elif "PostCode" in self.extra_data:
{'fraction_non_alphanumeric': 0.10399686178790916, 'fraction_numerical': 0.004009937671620974, 'mean_word_length': 2.238848108413326, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3055178', 'n_tokens_mistral': 6973, 'n_tokens_neox': 6592, 'n_words': 1458}