<commit_before>import os
import re

from django.contrib.auth.models import User
from django.db import models
from pyxform import SurveyElementBuilder
from pyxform.builder import create_survey_from_xls
from pyxform.question import Question
from pyxform.section import RepeatingSection

from common_tags import ID
from odk_logger.models import XForm
from odk_viewer.models import ParsedInstance
from odk_viewer.models.parsed_instance import _encode_for_mongo
from utils.export_tools import question_types_to_exclude, DictOrganizer
from utils.model_tools import queryset_iterator, set_uuid


class ColumnRename(models.Model):
    xpath = models.CharField(max_length=255, unique=True)
    column_name = models.CharField(max_length=32)

    class Meta:
        app_label = "odk_viewer"

    @classmethod
    def get_dict(cls):
        return dict([(cr.xpath, cr.column_name) for cr in cls.objects.all()])


def upload_to(instance, filename, username=None):
    if instance:
        username = instance.user.username
    return os.path.join(
        username,
        'xls',
        os.path.split(filename)[1]
        )


class DataDictionary(XForm):

    GEODATA_SUFFIXES = [
        'latitude',
        'longitude',
        'altitude',
        'precision'
    ]

    PREFIX_NAME_REGEX = re.compile(r'(?P<prefix>.+/)(?P<name>[^/]+)$')

    def __init__(self, *args, **kwargs):
        self.surveys_for_export = lambda d: d.surveys.all()
        super(DataDictionary, self).__init__(*args, **kwargs)

    def _set_uuid_in_xml(self):
        """
        Add bind to automatically set UUID node in XML.
        """
        file_name, file_ext = os.path.splitext(self.file_name())
        split_xml = self.uuid_regex.split(self.xml)
        split_xml[self.uuid_node_location:self.uuid_node_location] =\
            ['<formhub><uuid/></formhub>']
        split_xml[self.uuid_bind_location:self.uuid_bind_location] = [
            '\n      <bind nodeset="/', file_name,
            '/formhub/uuid" type="string" calculate="\'',
            self.uuid, '\'" />'
        ]
        self.xml = ''.join(split_xml)

    class Meta:
        app_label = "odk_viewer"
        proxy = True

    def add_surveys(self):
        if not hasattr(self, "_dict_organizer"):
            _dict_organizer = DictOrganizer()
        obs = []
        for d in self.get_list_of_parsed_instances(flat=False):
            obs.append(_dict_organizer.get_observation_from_dict(d))
        return obs

    def save(self, *args, **kwargs):
        if self.xls:
            survey = create_survey_from_xls(self.xls)
            self.json = survey.to_json()
            self.xml = survey.to_xml()
            self._mark_start_time_boolean()
            set_uuid(self)
            self._set_uuid_in_xml()
        super(DataDictionary, self).save(*args, **kwargs)

    def file_name(self):
        return os.path.split(self.xls.name)[-1]

    def get_survey(self):
        if not hasattr(self, "_survey"):
            builder = SurveyElementBuilder()
            self._survey = builder.create_survey_element_from_json(self.json)
        return self._survey

    survey = property(get_survey)

    def get_survey_elements(self):
        return self.survey.iter_descendants()

    def get_mongo_field_names_dict(self):
        """
        Return a dictionary of fieldnames as saved in mongodb with
        corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
        """
        names = {}
        for elem in self.get_survey_elements():
            names[_encode_for_mongo(unicode(elem.name))] = elem.name
        return names

    survey_elements = property(get_survey_elements)

    def xpath_of_first_geopoint(self):
        for e in self.get_survey_elements():
            if e.bind.get(u'type') == u'geopoint':
                return e.get_abbreviated_xpath()

    def has_surveys_with_geopoints(self):
        return ParsedInstance.objects.filter(instance__xform=self, lat__isnull=False).count() > 0

    def xpaths(self, prefix='', survey_element=None, result=None,
               repeat_iterations=4):
        """
        Return a list of XPaths for this survey that will be used as
        headers for the csv export.<commit_msg>get abbreviated xpath names<commit_after>import os
import re

from django.contrib.auth.models import User
from django.db import models
from pyxform import SurveyElementBuilder
from pyxform.builder import create_survey_from_xls
from pyxform.question import Question
from pyxform.section import RepeatingSection

from common_tags import ID
from odk_logger.models import XForm
from odk_viewer.models import ParsedInstance
from odk_viewer.models.parsed_instance import _encode_for_mongo
from utils.export_tools import question_types_to_exclude, DictOrganizer
from utils.model_tools import queryset_iterator, set_uuid


class ColumnRename(models.Model):
    xpath = models.CharField(max_length=255, unique=True)
    column_name = models.CharField(max_length=32)

    class Meta:
        app_label = "odk_viewer"

    @classmethod
    def get_dict(cls):
        return dict([(cr.xpath, cr.column_name) for cr in cls.objects.all()])


def upload_to(instance, filename, username=None):
    if instance:
        username = instance.user.username
    return os.path.join(
        username,
        'xls',
        os.path.split(filename)[1]
        )


class DataDictionary(XForm):

    GEODATA_SUFFIXES = [
        'latitude',
        'longitude',
        'altitude',
        'precision'
    ]

    PREFIX_NAME_REGEX = re.compile(r'(?P<prefix>.+/)(?P<name>[^/]+)$')

    def __init__(self, *args, **kwargs):
        self.surveys_for_export = lambda d: d.surveys.all()
        super(DataDictionary, self).__init__(*args, **kwargs)

    def _set_uuid_in_xml(self):
        """
        Add bind to automatically set UUID node in XML.
        """
        file_name, file_ext = os.path.splitext(self.file_name())
        split_xml = self.uuid_regex.split(self.xml)
        split_xml[self.uuid_node_location:self.uuid_node_location] =\
            ['<formhub><uuid/></formhub>']
        split_xml[self.uuid_bind_location:self.uuid_bind_location] = [
            '\n      <bind nodeset="/', file_name,
            '/formhub/uuid" type="string" calculate="\'',
            self.uuid, '\'" />'
        ]
        self.xml = ''.join(split_xml)

    class Meta:
        app_label = "odk_viewer"
        proxy = True

    def add_surveys(self):
        if not hasattr(self, "_dict_organizer"):
            _dict_organizer = DictOrganizer()
        obs = []
        for d in self.get_list_of_parsed_instances(flat=False):
            obs.append(_dict_organizer.get_observation_from_dict(d))
        return obs

    def save(self, *args, **kwargs):
        if self.xls:
            survey = create_survey_from_xls(self.xls)
            self.json = survey.to_json()
            self.xml = survey.to_xml()
            self._mark_start_time_boolean()
            set_uuid(self)
            self._set_uuid_in_xml()
        super(DataDictionary, self).save(*args, **kwargs)

    def file_name(self):
        return os.path.split(self.xls.name)[-1]

    def get_survey(self):
        if not hasattr(self, "_survey"):
            builder = SurveyElementBuilder()
            self._survey = builder.create_survey_element_from_json(self.json)
        return self._survey

    survey = property(get_survey)

    def get_survey_elements(self):
        return self.survey.iter_descendants()

    def get_mongo_field_names_dict(self):
        """
        Return a dictionary of fieldnames as saved in mongodb with
        corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
        """
        names = {}
        for elem in self.get_survey_elements():
            names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = \
                elem.get_abbreviated_xpath()
        return names

    survey_elements = property(get_survey_elements)

    def xpath_of_first_geopoint(self):
        for e in self.get_survey_elements():
            if e.bind.get(u'type') == u'geopoint':
                return e.get_abbreviated_xpath()

    def has_surveys_with_geopoints(self):
        return ParsedInstance.objects.filter(instance__xform=self, lat__isnull=False).count() > 0

    def xpaths(self, prefix='', survey_element=None, result=None,
               repeat_iterations=4):
        """
        Return a list of XPaths for this survey that will be used as
        headers for the csv export.
{'fraction_non_alphanumeric': 0.10487112792622369, 'fraction_numerical': 0.0030740127689761173, 'mean_word_length': 2.733009708737864, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7540742', 'n_tokens_mistral': 2718, 'n_tokens_neox': 2657, 'n_words': 542}