<commit_before># Python
import datetime
import dateutil.parser
import dateutil.relativedelta
import uuid
import random
from functools import wraps
import logging

# 3rd party
import numpy
import scipy.stats
import validictory
import json
import parse_python_indentation

# django core
from django.utils.translation import ugettext as _
import django.contrib.gis.db.models as gismodels
from django.db import models
from django.contrib.gis.geos import Point
from django.db.models import Q


# django 3rd party
from rest_framework.reverse import reverse
from phonenumber_field.modelfields import PhoneNumberField
from django_extensions.db.fields import (
    CreationDateTimeField, ModificationDateTimeField,
)
from django_pgjson.fields import JsonBField
from jsonfield import JSONField

# Internal
from fields import UniqueActiveField

logger = logging.getLogger(__name__)


class TeamMember(models.Model):
    MALE = 'M'
    FEMALE = 'F'
    GENDER_CHOICES = (
        (MALE, _('Male')),
        (FEMALE, _('Female')),
    )
    first_name = models.CharField(blank=False, max_length=50)
    last_name = models.CharField(blank=False, max_length=50)
    mobile = PhoneNumberField(blank=True)
    email = models.EmailField(blank=True)
    birth_year = models.IntegerField(blank=False)
    gender = models.CharField(choices=GENDER_CHOICES, max_length=3, blank=True)

    created = CreationDateTimeField()
    modified = ModificationDateTimeField()

    _last_survey = None

    def last_survey(self):
        # TODO: maybe get the reverse relationship instead

        # Poor mans cache
        if self._last_survey:
            return self._last_survey

        survey = (HouseholdSurveyJSON.objects.filter(
            Q(team_lead=self)
            | Q(team_assistant=self)
            | Q(team_anthropometrist=self)
        ).order_by('-id') or [None])[0]

        self._last_survey = survey
        return survey

    def last_survey_position(self):
        last_survey = self.last_survey()
        if last_survey:
            for position in ['team_lead', 'team_assistant', 'team_anthropometrist']:
                if self.id == getattr(last_survey, position).id:
                    return position

            raise "TeamMember has last_survey but no position, this shouldn't happen."

    def last_survey_created(self):
        last_survey = self.last_survey()
        if last_survey:
            return last_survey.get_start_time()

    def last_survey_cluster_name(self):
        last_survey = self.last_survey()
        if last_survey:
            return last_survey.json['cluster_name']

    def last_survey_cluster(self):
        last_survey = self.last_survey()
        if last_survey:
            return last_survey.json['cluster']

    class Meta:
        get_latest_by = 'modified'
        ordering = ('-modified', '-created',)

    @property
    def random_id(self):
        return random.randint(10000, 100000)

    def __unicode__(self):
        return u'%s-%s %s' % (self.id, self.first_name, self.last_name)

    def get_absolute_url(self):
        return reverse('teammember-detail', kwargs={'pk': str(self.pk)})


def validate_json(spec_file):
    survey_schema = None
    try:
        survey_schema = json.load(open(spec_file))
    except IOError as e:
        # log the error but it's ok if the spec is missing
        # if the validator is never called
        logger.exception(
            "Could not load the json spec %s. \nThis is ok if you are just setting up.\n%s" % (spec_file, e.message))

    @wraps(validate_json)
    def wrapped(value):
        assert survey_schema, "Trying to validate a non existant JSON schema file %s" % spec_file
        # It really should exist by now.
        validictory.validate(value, survey_schema, required_by_default=False)
    return wrapped


from django.db.models import F, Count, Func

class Age(Func):
    function = 'AGE'

class FullMonths(Func):
    # yes, it's sad that postgres doesn't have a `justify_months` fun
    template = '(( 12 * EXTRACT(years FROM %(expressions)s)) + EXTRACT(months FROM %(expressions)s))::integer'
    output_field = models.IntegerField()

class FullYears(Func):
    template = '(EXTRACT(years FROM %(expressions)s))::integer'
    output_field = models.IntegerField()

class HouseholdMemberManager(models.Manager):

    def by_teamlead(self, team_lead):
        return super(HouseholdMemberManager, self).get_queryset().filter(
            household_survey__team_lead=team_lead)

    def age_distribution_in_years(self):
        return super(HouseholdMemberManager, self).get_queryset().annotate(
            age_in_years=FullYears(Age(F('household_survey__start_time'), F('birthdate')))
        ).values('age_in_years').order_by('age_in_years').annotate(count=Count('age_in_years'))

class ChildrenManager(HouseholdMemberManager):

    def get_queryset(self):
        return super(ChildrenManager, self).get_queryset().annotate(
            age_in_years=FullYears(Age(F('household_survey__start_time'), F('birthdate')))
        ).filter(
            age_in_years__lt=6,
        )

    def age_distribution_in_months(self):
        return super(ChildrenManager, self).get_queryset().annotate(
            age_in_months=FullMonths(Age(F('household_survey__start_time'), F('birthdate')))
        ).values('age_in_months').order_by('age_in_months').annotate(count=Count('age_in_months'))

class WomenManager(HouseholdMemberManager):

    def get_queryset(self):
        return super(WomenManager, self).get_queryset().annotate(
            age_in_years=FullYears(Age(F('household_survey__start_time'), F('birthdate')))
        ).filter(
            age_in_years__gt=15,
            age_in_years__lt=49,
            gender='F',
        )


class BaseHouseholdMember(models.Model):

    # TODO add the following fields to the model that were present in the legacy
    # json field:
    # women:
    #   "breastfeeding",
    #   "age",
    #   "pregnant",
    #
    # children:<commit_msg>Make custom Manager/QuerySet functions queriable<commit_after># Python
import datetime
import dateutil.parser
import dateutil.relativedelta
import uuid
import random
from functools import wraps
import logging

# 3rd party
import numpy
import scipy.stats
import validictory
import json
import parse_python_indentation

# django core
from django.utils.translation import ugettext as _
import django.contrib.gis.db.models as gismodels
from django.db import models
from django.contrib.gis.geos import Point
from django.db.models import Q


# django 3rd party
from rest_framework.reverse import reverse
from phonenumber_field.modelfields import PhoneNumberField
from django_extensions.db.fields import (
    CreationDateTimeField, ModificationDateTimeField,
)
from django_pgjson.fields import JsonBField
from jsonfield import JSONField

# Internal
from fields import UniqueActiveField

logger = logging.getLogger(__name__)


class TeamMember(models.Model):
    MALE = 'M'
    FEMALE = 'F'
    GENDER_CHOICES = (
        (MALE, _('Male')),
        (FEMALE, _('Female')),
    )
    first_name = models.CharField(blank=False, max_length=50)
    last_name = models.CharField(blank=False, max_length=50)
    mobile = PhoneNumberField(blank=True)
    email = models.EmailField(blank=True)
    birth_year = models.IntegerField(blank=False)
    gender = models.CharField(choices=GENDER_CHOICES, max_length=3, blank=True)

    created = CreationDateTimeField()
    modified = ModificationDateTimeField()

    _last_survey = None

    def last_survey(self):
        # TODO: maybe get the reverse relationship instead

        # Poor mans cache
        if self._last_survey:
            return self._last_survey

        survey = (HouseholdSurveyJSON.objects.filter(
            Q(team_lead=self)
            | Q(team_assistant=self)
            | Q(team_anthropometrist=self)
        ).order_by('-id') or [None])[0]

        self._last_survey = survey
        return survey

    def last_survey_position(self):
        last_survey = self.last_survey()
        if last_survey:
            for position in ['team_lead', 'team_assistant', 'team_anthropometrist']:
                if self.id == getattr(last_survey, position).id:
                    return position

            raise "TeamMember has last_survey but no position, this shouldn't happen."

    def last_survey_created(self):
        last_survey = self.last_survey()
        if last_survey:
            return last_survey.get_start_time()

    def last_survey_cluster_name(self):
        last_survey = self.last_survey()
        if last_survey:
            return last_survey.json['cluster_name']

    def last_survey_cluster(self):
        last_survey = self.last_survey()
        if last_survey:
            return last_survey.json['cluster']

    class Meta:
        get_latest_by = 'modified'
        ordering = ('-modified', '-created',)

    @property
    def random_id(self):
        return random.randint(10000, 100000)

    def __unicode__(self):
        return u'%s-%s %s' % (self.id, self.first_name, self.last_name)

    def get_absolute_url(self):
        return reverse('teammember-detail', kwargs={'pk': str(self.pk)})


def validate_json(spec_file):
    survey_schema = None
    try:
        survey_schema = json.load(open(spec_file))
    except IOError as e:
        # log the error but it's ok if the spec is missing
        # if the validator is never called
        logger.exception(
            "Could not load the json spec %s. \nThis is ok if you are just setting up.\n%s" % (spec_file, e.message))

    @wraps(validate_json)
    def wrapped(value):
        assert survey_schema, "Trying to validate a non existant JSON schema file %s" % spec_file
        # It really should exist by now.
        validictory.validate(value, survey_schema, required_by_default=False)
    return wrapped


from django.db.models import F, Count, Func

class Age(Func):
    function = 'AGE'

class FullMonths(Func):
    # yes, it's sad that postgres doesn't have a `justify_months` fun
    template = '(( 12 * EXTRACT(years FROM %(expressions)s)) + EXTRACT(months FROM %(expressions)s))::integer'
    output_field = models.IntegerField()

class FullYears(Func):
    template = '(EXTRACT(years FROM %(expressions)s))::integer'
    output_field = models.IntegerField()

class HouseholdMemberQuerySet(models.QuerySet):

    def by_teamlead(self, team_lead):
        return self.filter(household_survey__team_lead=team_lead)

    def age_distribution_in_years(self):
        return self.annotate(
            age_in_years=FullYears(Age(F('household_survey__start_time'), F('birthdate')))
        ).values('age_in_years').order_by('age_in_years').annotate(count=Count('age_in_years'))

    def age_distribution_in_months(self):
        return self.annotate(
            age_in_months=FullMonths(Age(F('household_survey__start_time'), F('birthdate')))
        ).values('age_in_months').order_by('age_in_months').annotate(count=Count('age_in_months'))

HouseholdMemberManager = models.Manager.from_queryset(HouseholdMemberQuerySet)

class ChildrenManager(HouseholdMemberManager):

    def get_queryset(self):
        return super(ChildrenManager, self).get_queryset().annotate(
            age_in_years=FullYears(Age(F('household_survey__start_time'), F('birthdate')))
        ).filter(
            age_in_years__lt=6,
        )

class WomenManager(HouseholdMemberManager):

    def get_queryset(self):
        return super(WomenManager, self).get_queryset().annotate(
            age_in_years=FullYears(Age(F('household_survey__start_time'), F('birthdate')))
        ).filter(
            age_in_years__gt=15,
            age_in_years__lt=49,
            gender='F',
        )


class BaseHouseholdMember(models.Model):

    # TODO add the following fields to the model that were present in the legacy
    # json field:
    # women:
    #   "breastfeeding",
    #   "age",
    #   "pregnant",
    #
    # children:
{'fraction_non_alphanumeric': 0.0981656755172125, 'fraction_numerical': 0.0043554736577602815, 'mean_word_length': 3.3213897937024974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4411180', 'n_tokens_mistral': 3824, 'n_tokens_neox': 3628, 'n_words': 821}