<commit_before>from django.db import models
from django.conf import settings

from xform_manager.models import XForm, Instance
from phone_manager.models import Phone
from surveyor_manager.models import Surveyor
from locations.models import District
from nga_districts.models import LGA

from xform_manager import utils
from common_tags import IMEI, DEVICE_ID, START_TIME, START, \
    END_TIME, END, LGA_ID, ID, SURVEYOR_NAME, ATTACHMENTS, DATE
import django.dispatch
import datetime

from sentry.client.models import client as sentry_client
import logging

# this is Mongo Collection (SQL table equivalent) where we will store
# the parsed submissions
xform_instances = settings.MONGO_DB.instances

class ParseError(Exception):
    pass

def datetime_from_str(text):
    # Assumes text looks like 2011-01-01T09:50:06.966
    if text is None: return None
    date_time_str = text.split(".")[0]
    return datetime.datetime.strptime(
        date_time_str, '%Y-%m-%dT%H:%M:%S'
        )

class ParsedInstance(models.Model):
    instance = models.OneToOneField(Instance, related_name="parsed_instance")
    phone = models.ForeignKey(Phone, null=True)
    surveyor = models.ForeignKey(Surveyor, null=True)
    
    # district is no longer used except in old data. once
    # we've migrated phase I surveys, we should delete this field.
    district = models.ForeignKey(District, null=True)
    lga = models.ForeignKey(LGA, null=True)
    start_time = models.DateTimeField(null=True)
    end_time = models.DateTimeField(null=True)
    surveyor = models.ForeignKey(Surveyor, null=True)
    is_new = models.BooleanField(default=False)
    
    class Meta:
        app_label = "parsed_xforms"
    
    def to_dict(self):
        if not hasattr(self, "_dict_cache"):
            self._dict_cache = self.instance.get_dict()
        self._dict_cache.update(
            {
                ID : self.get_mongo_id(),
                SURVEYOR_NAME :
                    None if not self.surveyor else self.surveyor.name,
                LGA_ID :
                    None if not self.lga else self.lga.id,
                ATTACHMENTS :
                    [a.media_file.name for a in self.instance.attachments.all()],
                }
            )
        return self._dict_cache
    
    def get_mongo_id(self):
        return self.instance.id

    def _set_phone(self):
        doc = self.to_dict()
        # I'm using two different keys here because I switched the key
        # we're using in Phase II. Ideally, we'd do this using a data
        # dictionary.
        if (IMEI in doc) or (DEVICE_ID in doc):
            imei = doc.get(IMEI, doc.get(DEVICE_ID))
            if imei is None:
                self.phone = None
            else:
                self.phone, created = Phone.objects.get_or_create(imei=imei)

    def _set_start_time(self):
        doc = self.to_dict()
        if START_TIME in doc:
            date_time_str = doc[START_TIME]
            self.start_time = datetime_from_str(date_time_str)
        elif START in doc:
            date_time_str = doc[START]
            self.start_time = datetime_from_str(date_time_str)
        else:
            self.start_time = None
<commit_msg>Added instance.status field to mongo, this important so m and e knows which surveys are complete.<commit_after>from django.db import models
from django.conf import settings

from xform_manager.models import XForm, Instance
from phone_manager.models import Phone
from surveyor_manager.models import Surveyor
from locations.models import District
from nga_districts.models import LGA

from xform_manager import utils
from common_tags import IMEI, DEVICE_ID, START_TIME, START, \
    END_TIME, END, LGA_ID, ID, SURVEYOR_NAME, ATTACHMENTS, DATE
import django.dispatch
import datetime

from sentry.client.models import client as sentry_client
import logging

# this is Mongo Collection (SQL table equivalent) where we will store
# the parsed submissions
xform_instances = settings.MONGO_DB.instances

class ParseError(Exception):
    pass

def datetime_from_str(text):
    # Assumes text looks like 2011-01-01T09:50:06.966
    if text is None: return None
    date_time_str = text.split(".")[0]
    return datetime.datetime.strptime(
        date_time_str, '%Y-%m-%dT%H:%M:%S'
        )

class ParsedInstance(models.Model):
    instance = models.OneToOneField(Instance, related_name="parsed_instance")
    phone = models.ForeignKey(Phone, null=True)
    surveyor = models.ForeignKey(Surveyor, null=True)
    
    # district is no longer used except in old data. once
    # we've migrated phase I surveys, we should delete this field.
    district = models.ForeignKey(District, null=True)
    lga = models.ForeignKey(LGA, null=True)
    start_time = models.DateTimeField(null=True)
    end_time = models.DateTimeField(null=True)
    surveyor = models.ForeignKey(Surveyor, null=True)
    is_new = models.BooleanField(default=False)
    
    class Meta:
        app_label = "parsed_xforms"
    
    def to_dict(self):
        if not hasattr(self, "_dict_cache"):
            self._dict_cache = self.instance.get_dict()
        self._dict_cache.update(
            {
                ID : self.get_mongo_id(),
                SURVEYOR_NAME :
                    None if not self.surveyor else self.surveyor.name,
                LGA_ID :
                    None if not self.lga else self.lga.id,
                ATTACHMENTS :
                    [a.media_file.name for a in self.instance.attachments.all()],
                u"_status" : self.instance.status,
                }
            )
        return self._dict_cache
    
    def get_mongo_id(self):
        return self.instance.id

    def _set_phone(self):
        doc = self.to_dict()
        # I'm using two different keys here because I switched the key
        # we're using in Phase II. Ideally, we'd do this using a data
        # dictionary.
        if (IMEI in doc) or (DEVICE_ID in doc):
            imei = doc.get(IMEI, doc.get(DEVICE_ID))
            if imei is None:
                self.phone = None
            else:
                self.phone, created = Phone.objects.get_or_create(imei=imei)

    def _set_start_time(self):
        doc = self.to_dict()
        if START_TIME in doc:
            date_time_str = doc[START_TIME]
            self.start_time = datetime_from_str(date_time_str)
        elif START in doc:
            date_time_str = doc[START]
            self.start_time = datetime_from_str(date_time_str)
        else:
            self.start_time = None

{'fraction_non_alphanumeric': 0.08038585209003216, 'fraction_numerical': 0.005512172714745062, 'mean_word_length': 2.506172839506173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5436698', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1954, 'n_words': 529}