<commit_before>import cPickle
import json
import os.path
import re
import uuid

from django.contrib.gis.db import models
from django.contrib.postgres.fields import JSONField
from django.contrib.postgres.fields import ArrayField
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from anymail.signals import EventType

from common.utils import nhs_titlecase
from dmd.models import DMDProduct
from frontend.managers import MeasureValueQuerySet
from frontend.validators import isAlphaNumeric
from frontend import model_prescribing_units


def _load_measure_tags(filename):
    with open(filename) as f:
        tags = json.load(f)
    for tag in tags.values():
        if isinstance(tag.get('description'), list):
            tag['description'] = ''.join(tag['description'])
    return tags


MEASURE_TAGS = _load_measure_tags(
    os.path.join(os.path.dirname(__file__), '../common/measure_tags.json'))


class Section(models.Model):
    bnf_id = models.CharField(max_length=8, primary_key=True)
    name = models.CharField(max_length=200)
    number_str = models.CharField(max_length=12)
    bnf_chapter = models.IntegerField()
    bnf_section = models.IntegerField(null=True, blank=True)
    bnf_para = models.IntegerField(null=True, blank=True)
    is_current = models.BooleanField(default=True)

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        self.number_str = self.get_number_str(self.bnf_id)
        super(Section, self).save(*args, **kwargs)

    def get_number_str(self, id):
        s1 = self.strip_zeros(id[:2])
        s2 = self.strip_zeros(id[2:4])
        s3 = self.strip_zeros(id[4:6])
        s4 = self.strip_zeros(id[6:8])
        number = str(s1)
        if s2:
            number += '.%s' % s2
        if s3:
            number += '.%s' % s3
        if s4:
            number += '.%s' % s4
        return number

    def strip_zeros(self, str):
        if not str or str == '0' or str == '00':
            return None
        if len(str) > 1 and str[0] == '0':
            str = str[1:]
        return int(str)

    class Meta:
        ordering = ["bnf_id"]


class RegionalTeam(models.Model):
    code = models.CharField(max_length=3, primary_key=True)
    name = models.CharField(max_length=200, null=True, blank=True)
    open_date = models.DateField(null=True, blank=True)
    close_date = models.DateField(null=True, blank=True)
    address = models.CharField(max_length=400, null=True, blank=True)
    postcode = models.CharField(max_length=10, null=True, blank=True)

    def __str__(self):
        return self.name


class STP(models.Model):
    ons_code = models.CharField(max_length=9, primary_key=True)
    name = models.CharField(max_length=200, null=True, blank=True)

    def __str__(self):
        return self.name


class PCT(models.Model):
    '''
    PCTs or CCGs (depending on date).
    '''
    PCT_ORG_TYPES = (
        ('CCG', 'CCG'),
        ('PCT', 'PCT'),
        ('H', 'Hub'),
        ('Unknown', 'Unknown')
    )
    code = models.CharField(max_length=3, primary_key=True,
                            help_text='Primary care trust code')
    regional_team = models.ForeignKey(RegionalTeam, null=True)
    stp = models.ForeignKey(STP, null=True)
    ons_code = models.CharField(max_length=9, null=True, blank=True)
    name = models.CharField(max_length=200, null=True, blank=True)
    org_type = models.CharField(max_length=9, choices=PCT_ORG_TYPES,
                                default='Unknown')
    boundary = models.GeometryField(null=True, blank=True, srid=4326)
    centroid = models.PointField(null=True, blank=True)
    open_date = models.DateField(null=True, blank=True)
    close_date = models.DateField(null=True, blank=True)
    address = models.CharField(max_length=400, null=True, blank=True)
    postcode = models.CharField(max_length=10, null=True, blank=True)

    objects = models.GeoManager()<commit_msg>Add comment about NULLable fields on PCT<commit_after>import cPickle
import json
import os.path
import re
import uuid

from django.contrib.gis.db import models
from django.contrib.postgres.fields import JSONField
from django.contrib.postgres.fields import ArrayField
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from anymail.signals import EventType

from common.utils import nhs_titlecase
from dmd.models import DMDProduct
from frontend.managers import MeasureValueQuerySet
from frontend.validators import isAlphaNumeric
from frontend import model_prescribing_units


def _load_measure_tags(filename):
    with open(filename) as f:
        tags = json.load(f)
    for tag in tags.values():
        if isinstance(tag.get('description'), list):
            tag['description'] = ''.join(tag['description'])
    return tags


MEASURE_TAGS = _load_measure_tags(
    os.path.join(os.path.dirname(__file__), '../common/measure_tags.json'))


class Section(models.Model):
    bnf_id = models.CharField(max_length=8, primary_key=True)
    name = models.CharField(max_length=200)
    number_str = models.CharField(max_length=12)
    bnf_chapter = models.IntegerField()
    bnf_section = models.IntegerField(null=True, blank=True)
    bnf_para = models.IntegerField(null=True, blank=True)
    is_current = models.BooleanField(default=True)

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        self.number_str = self.get_number_str(self.bnf_id)
        super(Section, self).save(*args, **kwargs)

    def get_number_str(self, id):
        s1 = self.strip_zeros(id[:2])
        s2 = self.strip_zeros(id[2:4])
        s3 = self.strip_zeros(id[4:6])
        s4 = self.strip_zeros(id[6:8])
        number = str(s1)
        if s2:
            number += '.%s' % s2
        if s3:
            number += '.%s' % s3
        if s4:
            number += '.%s' % s4
        return number

    def strip_zeros(self, str):
        if not str or str == '0' or str == '00':
            return None
        if len(str) > 1 and str[0] == '0':
            str = str[1:]
        return int(str)

    class Meta:
        ordering = ["bnf_id"]


class RegionalTeam(models.Model):
    code = models.CharField(max_length=3, primary_key=True)
    name = models.CharField(max_length=200, null=True, blank=True)
    open_date = models.DateField(null=True, blank=True)
    close_date = models.DateField(null=True, blank=True)
    address = models.CharField(max_length=400, null=True, blank=True)
    postcode = models.CharField(max_length=10, null=True, blank=True)

    def __str__(self):
        return self.name


class STP(models.Model):
    ons_code = models.CharField(max_length=9, primary_key=True)
    name = models.CharField(max_length=200, null=True, blank=True)

    def __str__(self):
        return self.name


class PCT(models.Model):
    '''
    PCTs or CCGs (depending on date).
    '''
    PCT_ORG_TYPES = (
        ('CCG', 'CCG'),
        ('PCT', 'PCT'),
        ('H', 'Hub'),
        ('Unknown', 'Unknown')
    )
    code = models.CharField(max_length=3, primary_key=True,
                            help_text='Primary care trust code')

    # These are NULLable, because not every PCT belongs to either a
    # RegionalTeam or an STP.  Specifically, we create PCT objects when
    # importing prescribing data if the PCT is not otherwise in our database.
    regional_team = models.ForeignKey(RegionalTeam, null=True)
    stp = models.ForeignKey(STP, null=True)

    ons_code = models.CharField(max_length=9, null=True, blank=True)
    name = models.CharField(max_length=200, null=True, blank=True)
    org_type = models.CharField(max_length=9, choices=PCT_ORG_TYPES,
                                default='Unknown')
    boundary = models.GeometryField(null=True, blank=True, srid=4326)
    centroid = models.PointField(null=True, blank=True)
    open_date = models.DateField(null=True, blank=True)
    close_date = models.DateField(null=True, blank=True)
    address = models.CharField(max_length=400, null=True, blank=True)
    postcode = models.CharField(max_length=10, null=True, blank=True)

    objects = models.GeoManager()
{'fraction_non_alphanumeric': 0.11745254133496633, 'fraction_numerical': 0.014451928965094916, 'mean_word_length': 3.2620041753653446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1989490', 'n_tokens_mistral': 2732, 'n_tokens_neox': 2642, 'n_words': 582}