<commit_before>from datetime import date, datetime
import os
import pytz
from lxml import etree
from dateutil.parser import parse as extract_date
from django.forms import ValidationError
from django.http import HttpResponse, HttpResponseNotFound, StreamingHttpResponse
from django.core.servers.basehttp import FileWrapper
from django.core.files.storage import get_storage_class
from django.conf import settings
from django import template
from django.utils import timezone
from djangohttpdigest.digest import Digestor
from djangohttpdigest.authentication import SimpleHardcodedAuthenticator
from django.utils.translation import ugettext as _
from survey.models import (Survey, Interviewer, Interview, SurveyAllocation, ODKAccess, QuestionSet,
                           Question, Batch, ODKSubmission, ODKGeoPoint, TextAnswer, Answer, NonResponseAnswer,
                           VideoAnswer, AudioAnswer, ImageAnswer, MultiSelectAnswer, MultiChoiceAnswer, DateAnswer,
                           GeopointAnswer, ListingSample)
from survey.odk.utils.log import logger
from functools import wraps
from survey.utils.zip import InMemoryZip
from django.contrib.sites.models import Site
from django_rq import job, get_connection


OPEN_ROSA_VERSION_HEADER = 'X-OpenRosa-Version'
HTTP_OPEN_ROSA_VERSION_HEADER = 'HTTP_X_OPENROSA_VERSION'
OPEN_ROSA_VERSION = '1.0'
DEFAULT_CONTENT_TYPE = 'text/xml; charset=utf-8'
INSTANCE_ID_PATH = '//qset/meta/instanceID'
INSTANCE_NAME_PATH = '//qset/meta/instanceName'
DEFAULT_DATE_CREATED_PATH = '//qset/meta/creationDate'
FORM_ID_PATH = '//qset/@id'
SUBMISSIONS_ID_PATH = '//qset/submissions/id'
FORM_TYPE_PATH = '//qset/type'
NON_RESPONSE_CONFIRMATION_PATH = '//qset/nqrc'
NON_RESPONSE_PATH = '//qset/nqr'
NON_RESPONSE_OTHERS_PATH = '//qset/nqr_other'
FORM_ASSIGNMENT_PATH = '//qset/surveyAllocation'
ANSWER_NODE_PATH = '//qset/qset{{ qset_id }}'
# default content length for submission requests
DEFAULT_CONTENT_LENGTH = 10000000
MAX_DISPLAY_PER_COLLECTOR = 1000


class NotEnoughData(ValueError):
    pass


def _get_tree_from_blob(blob_contents):
    return etree.fromstring(blob_contents)


def _get_tree(xml_file):
    return etree.fromstring(xml_file.read())


# either tree or xml_string must be defined
def _get_nodes(search_path, tree=None, xml_string=None):
    if tree is None:
        tree = etree.fromstring(xml_string)
    try:
        return tree.xpath(search_path)
    except Exception, ex:
        logger.error('Error retrieving path: %s, Desc: %s' %
                     (search_path, str(ex)))


@job('odk', connection=get_connection())
def process_answers(xml, qset, access_channel, question_map, survey_allocation, submission, media_files):
    """Process answers for this answers_node. It's supposed to handle for all question answers in this xform.
    :param answers_node:
    :param qset:
    :param interviewer:
    :param question_map:
    :param survey_allocation:
    :return:
    """
    survey_tree = _get_tree_from_blob(xml)
    answers_nodes = _get_answer_nodes(survey_tree, qset)
    created_interviews = []
    for answers_node in answers_nodes:
        # answers = []
        survey_parameters = []
        survey = survey_allocation.survey
        question_answers_node = _get_nodes('./questions/surveyQuestions', answers_node)[0]
        reference_interview = None          # typically used if
        if _get_nodes('./sampleData/selectedSample', answers_node):
            # the following looks ugly but ./sampleData/selectedSample is calculated in xform by a concat of
            # sampleData/iq{{ ea_id }} values with -. See question_set.xml binding for ./sampleData/selectedSample
            # if for some reason more than one interview value is reflected, choose the first one
            reference_interview = _get_nodes('./sampleData/selectedSample',
                                             answers_node)[0].text.strip('-').split('-')[0]
        # map(lambda node: answers.extend(get_answers(node, qset, question_map)), question_answers_node.getchildren())
        # map(lambda node: survey_parameters.extend(get_answers(node, qset, question_map)),
        #     survey_parameters_node.getchildren())
        # now check if non response exists and is selected
        if _get_nodes(NON_RESPONSE_CONFIRMATION_PATH) and int(_get_nodes(NON_RESPONSE_CONFIRMATION_PATH)[0].text) > 0:
            answer = _get_nodes(NON_RESPONSE_PATH)[0].text
            if answer.upper() == 'OTHER':        # user selected non respons
                answer = _get_nodes(NON_RESPONSE_OTHERS_PATH)[0].text
            non_response = save_non_response(survey_tree, qset, survey, survey_allocation,
                                             access_channel, answer, reference_interview)
            created_interviews.append(non_response.interview)
        else:
            answers = get_answers(question_answers_node, qset, question_map, _get_default_date_created(survey_tree))
            survey_parameters = None
            if hasattr(qset, 'parameter_list'):
                survey_parameters_node = _get_nodes('./questions/groupQuestions', answers_node)[0]
                # survey paramaters does not have any single repeat
                survey_parameters = get_answers(survey_parameters_node, qset, question_map,
                                                _get_default_date_created(survey_tree))[0]
            if survey_allocation.stage in [None, SurveyAllocation.LISTING] and \
                    survey.has_sampling and survey.sample_size > len(answers):
                raise NotEnoughData()
            created_interviews.extend(Interview.save_answers(qset, survey, survey_allocation.allocation_ea,
                                                             access_channel, question_map, answers,
                                                             survey_parameters=survey_parameters,
                                                             reference_interview=reference_interview,
                                                             media_files=media_files))
        if survey.has_sampling:
            survey_allocation.stage = SurveyAllocation.SURVEY
            survey_allocation.save()
    submission.status = ODKSubmission.COMPLETED
    submission.interviews.all().delete()          # wipe off the old interviews for this submission
    map(lambda interview: submission.interviews.add(interview), created_interviews)    # update with present interviews
    submission.save()
    submission.save_attachments(media_files)


def save_non_response(survey_tree, qset, survey, survey_allocation, access_channel, answer, reference_interview):
    interviewer = survey_allocation.interviewer
    interview = Interview.objects.create(survey=survey, question_set=qset, ea=survey_allocation.allocation_ea,
                                         interviewer=interviewer, interview_channel=access_channel,
                                         closure_date=extract_date(_get_nodes('./creationDate',
                                                                              tree=survey_tree)[0].text,
                                                                   dayfirst=False),
                                         interview_reference_id=reference_interview)
    return NonResponseAnswer.objects.create(interview=interview, value=answer, interviewer=interviewer)



def get_answers(node, qset, question_map, completion_date):
    """get answers for the node set. Would work for nested loops but for loops sitting in same inline question thread
    """
    answers = []
    inline_record = {}
    for e in node.getchildren():
        if e.getchildren():
            if _get_nodes('./creationDate', tree=e):
                completion_date = extract_date(_get_nodes('./creationDate', tree=e)[0].text, dayfirst=False)
            loop_answers = get_answers(e, qset, question_map, completion_date)
            _update_loop_answers(inline_record, loop_answers)
            answers.extend(loop_answers)
        else:
            inline_record['completion_date'] = completion_date<commit_msg>non-response submission issue<commit_after>from datetime import date, datetime
import os
import pytz
from lxml import etree
from dateutil.parser import parse as extract_date
from django.forms import ValidationError
from django.http import HttpResponse, HttpResponseNotFound, StreamingHttpResponse
from django.core.servers.basehttp import FileWrapper
from django.core.files.storage import get_storage_class
from django.conf import settings
from django import template
from django.utils import timezone
from djangohttpdigest.digest import Digestor
from djangohttpdigest.authentication import SimpleHardcodedAuthenticator
from django.utils.translation import ugettext as _
from survey.models import (Survey, Interviewer, Interview, SurveyAllocation, ODKAccess, QuestionSet,
                           Question, Batch, ODKSubmission, ODKGeoPoint, TextAnswer, Answer, NonResponseAnswer,
                           VideoAnswer, AudioAnswer, ImageAnswer, MultiSelectAnswer, MultiChoiceAnswer, DateAnswer,
                           GeopointAnswer, ListingSample)
from survey.odk.utils.log import logger
from functools import wraps
from survey.utils.zip import InMemoryZip
from django.contrib.sites.models import Site
from django_rq import job, get_connection


OPEN_ROSA_VERSION_HEADER = 'X-OpenRosa-Version'
HTTP_OPEN_ROSA_VERSION_HEADER = 'HTTP_X_OPENROSA_VERSION'
OPEN_ROSA_VERSION = '1.0'
DEFAULT_CONTENT_TYPE = 'text/xml; charset=utf-8'
INSTANCE_ID_PATH = '//qset/meta/instanceID'
INSTANCE_NAME_PATH = '//qset/meta/instanceName'
CREATION_DATE_PATH = '//qset/meta/creationDate'
DEFAULT_DATE_CREATED_PATH = '//qset/meta/creationDate'
FORM_ID_PATH = '//qset/@id'
SUBMISSIONS_ID_PATH = '//qset/submissions/id'
FORM_TYPE_PATH = '//qset/type'
NON_RESPONSE_CONFIRMATION_PATH = '//qset/nqrc'
NON_RESPONSE_PATH = '//qset/nqr'
NON_RESPONSE_OTHERS_PATH = '//qset/nqr_other'
FORM_ASSIGNMENT_PATH = '//qset/surveyAllocation'
ANSWER_NODE_PATH = '//qset/qset{{ qset_id }}'
# default content length for submission requests
DEFAULT_CONTENT_LENGTH = 10000000
MAX_DISPLAY_PER_COLLECTOR = 1000


class NotEnoughData(ValueError):
    pass


def _get_tree_from_blob(blob_contents):
    return etree.fromstring(blob_contents)


def _get_tree(xml_file):
    return etree.fromstring(xml_file.read())


# either tree or xml_string must be defined
def _get_nodes(search_path, tree=None, xml_string=None):
    if tree is None:
        tree = etree.fromstring(xml_string)
    try:
        return tree.xpath(search_path)
    except Exception, ex:
        logger.error('Error retrieving path: %s, Desc: %s' %
                     (search_path, str(ex)))


@job('odk', connection=get_connection())
def process_answers(xml, qset, access_channel, question_map, survey_allocation, submission, media_files):
    """Process answers for this answers_node. It's supposed to handle for all question answers in this xform.
    :param answers_node:
    :param qset:
    :param interviewer:
    :param question_map:
    :param survey_allocation:
    :return:
    """
    survey_tree = _get_tree_from_blob(xml)
    answers_nodes = _get_answer_nodes(survey_tree, qset)
    created_interviews = []
    for answers_node in answers_nodes:
        # answers = []
        survey_parameters = []
        survey = survey_allocation.survey
        question_answers_node = _get_nodes('./questions/surveyQuestions', answers_node)[0]
        reference_interview = None          # typically used if
        if _get_nodes('./sampleData/selectedSample', answers_node):
            # the following looks ugly but ./sampleData/selectedSample is calculated in xform by a concat of
            # sampleData/iq{{ ea_id }} values with -. See question_set.xml binding for ./sampleData/selectedSample
            # if for some reason more than one interview value is reflected, choose the first one
            reference_interview = _get_nodes('./sampleData/selectedSample',
                                             answers_node)[0].text.strip('-').split('-')[0]
        # map(lambda node: answers.extend(get_answers(node, qset, question_map)), question_answers_node.getchildren())
        # map(lambda node: survey_parameters.extend(get_answers(node, qset, question_map)),
        #     survey_parameters_node.getchildren())
        # now check if non response exists and is selected
        if _get_nodes(NON_RESPONSE_CONFIRMATION_PATH) and int(_get_nodes(NON_RESPONSE_CONFIRMATION_PATH)[0].text) > 0:
            answer = _get_nodes(NON_RESPONSE_PATH)[0].text
            if answer.upper() == 'OTHER':        # user selected non respons
                answer = _get_nodes(NON_RESPONSE_OTHERS_PATH)[0].text
            non_response = save_non_response(survey_tree, qset, survey, survey_allocation,
                                             access_channel, answer, reference_interview)
            created_interviews.append(non_response.interview)
        else:
            answers = get_answers(question_answers_node, qset, question_map, _get_default_date_created(survey_tree))
            survey_parameters = None
            if hasattr(qset, 'parameter_list'):
                survey_parameters_node = _get_nodes('./questions/groupQuestions', answers_node)[0]
                # survey paramaters does not have any single repeat
                survey_parameters = get_answers(survey_parameters_node, qset, question_map,
                                                _get_default_date_created(survey_tree))[0]
            if survey_allocation.stage in [None, SurveyAllocation.LISTING] and \
                    survey.has_sampling and survey.sample_size > len(answers):
                raise NotEnoughData()
            created_interviews.extend(Interview.save_answers(qset, survey, survey_allocation.allocation_ea,
                                                             access_channel, question_map, answers,
                                                             survey_parameters=survey_parameters,
                                                             reference_interview=reference_interview,
                                                             media_files=media_files))
        if survey.has_sampling:
            survey_allocation.stage = SurveyAllocation.SURVEY
            survey_allocation.save()
    submission.status = ODKSubmission.COMPLETED
    submission.interviews.all().delete()          # wipe off the old interviews for this submission
    map(lambda interview: submission.interviews.add(interview), created_interviews)    # update with present interviews
    submission.save()
    submission.save_attachments(media_files)


def save_non_response(survey_tree, qset, survey, survey_allocation, access_channel, answer, reference_interview):
    interviewer = survey_allocation.interviewer
    interview = Interview.objects.create(survey=survey, question_set=qset, ea=survey_allocation.allocation_ea,
                                         interviewer=interviewer, interview_channel=access_channel,
                                         closure_date=extract_date(_get_nodes(CREATION_DATE_PATH,
                                                                              tree=survey_tree)[0].text,
                                                                   dayfirst=False),
                                         interview_reference_id=reference_interview)
    return NonResponseAnswer.objects.create(interview=interview, value=answer, interviewer=interviewer)



def get_answers(node, qset, question_map, completion_date):
    """get answers for the node set. Would work for nested loops but for loops sitting in same inline question thread
    """
    answers = []
    inline_record = {}
    for e in node.getchildren():
        if e.getchildren():
            if _get_nodes('./creationDate', tree=e):
                completion_date = extract_date(_get_nodes('./creationDate', tree=e)[0].text, dayfirst=False)
            loop_answers = get_answers(e, qset, question_map, completion_date)
            _update_loop_answers(inline_record, loop_answers)
            answers.extend(loop_answers)
        else:
            inline_record['completion_date'] = completion_date
{'fraction_non_alphanumeric': 0.0780002456700651, 'fraction_numerical': 0.0031937108463333743, 'mean_word_length': 2.8, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1339365', 'n_tokens_mistral': 4531, 'n_tokens_neox': 4394, 'n_words': 981}