<commit_before>from datetime import date, datetime
import os
import pytz
import re
import tempfile
import traceback
from xml.dom import Node
from xml.parsers.expat import ExpatError

from django.conf import settings
from django.core.exceptions import ValidationError, PermissionDenied
from django.core.files.storage import get_storage_class
from django.core.mail import mail_admins
from django.core.servers.basehttp import FileWrapper
from django.contrib.auth.models import User
from django.db import IntegrityError, transaction
from django.db.models.signals import pre_delete
from django.http import HttpResponse, HttpResponseNotFound, \
    StreamingHttpResponse
from django.shortcuts import get_object_or_404
from django.utils.encoding import DjangoUnicodeDecodeError
from django.utils.translation import ugettext as _
from django.utils import timezone
from modilabs.utils.subprocess_timeout import ProcessTimedOut
from pyxform.errors import PyXFormError
from pyxform.xform2json import create_survey_element_from_xml
import sys

from onadata.apps.logger.models import Attachment
from onadata.apps.logger.models import Instance
from onadata.apps.logger.models.instance import (
    FormInactiveError,
    InstanceHistory,
    get_id_string_from_xml_str)
from onadata.apps.logger.models import XForm
from onadata.apps.logger.models.xform import XLSFormError
from onadata.apps.logger.xform_instance_parser import (
    InstanceEmptyError,
    InstanceInvalidUserError,
    InstanceMultipleNodeError,
    DuplicateInstance,
    clean_and_parse_xml,
    get_uuid_from_xml,
    get_deprecated_uuid_from_xml,
    get_submission_date_from_xml)
from onadata.apps.viewer.models.data_dictionary import DataDictionary
from onadata.apps.viewer.models.parsed_instance import _remove_from_mongo,\
    xform_instances, ParsedInstance
from onadata.libs.utils import common_tags
from onadata.libs.utils.model_tools import queryset_iterator, set_uuid


OPEN_ROSA_VERSION_HEADER = 'X-OpenRosa-Version'
HTTP_OPEN_ROSA_VERSION_HEADER = 'HTTP_X_OPENROSA_VERSION'
OPEN_ROSA_VERSION = '1.0'
DEFAULT_CONTENT_TYPE = 'text/xml; charset=utf-8'
DEFAULT_CONTENT_LENGTH = settings.DEFAULT_CONTENT_LENGTH

uuid_regex = re.compile(r'<formhub><uuid>([^<]+)</uuid></formhub>',
                        re.DOTALL)

mongo_instances = settings.MONGO_DB.instances


def _get_instance(xml, new_uuid, submitted_by, status, xform):
    # check if its an edit submission
    old_uuid = get_deprecated_uuid_from_xml(xml)
    instances = Instance.objects.filter(uuid=old_uuid)

    if instances:
        # edits
        check_edit_submission_permissions(submitted_by, xform)
        instance = instances[0]
        InstanceHistory.objects.create(
            xml=instance.xml, xform_instance=instance, uuid=old_uuid)
        instance.xml = xml
        instance.uuid = new_uuid
        instance.save()
    else:
        # new submission
        instance = Instance.objects.create(
            xml=xml, user=submitted_by, status=status, xform=xform)

    return instance


def get_uuid_from_submission(xml):
    # parse UUID from uploaded XML
    split_xml = uuid_regex.split(xml)

    # check that xml has UUID
    return len(split_xml) > 1 and split_xml[1] or None


def get_xform_from_submission(xml, username, uuid=None):
        # check alternative form submission ids
        uuid = uuid or get_uuid_from_submission(xml)

        if not username and not uuid:
            raise InstanceInvalidUserError()

        if uuid:
            # try find the form by its uuid which is the ideal condition
            if XForm.objects.filter(uuid=uuid).count() > 0:
                xform = XForm.objects.get(uuid=uuid)
                xform_username = xform.user.username

                if xform_username != username:
                    raise InstanceInvalidUserError()

                return xform

        id_string = get_id_string_from_xml_str(xml)

        return get_object_or_404(XForm, id_string__iexact=id_string,
                                 user__username=username)


def _has_edit_xform_permission(xform, user):
    if isinstance(xform, XForm) and isinstance(user, User):
        return user.has_perm('logger.change_xform', xform)

    return False


def check_edit_submission_permissions(request_user, xform):
    if xform and request_user and request_user.is_authenticated():
        requires_auth = xform.user.profile.require_auth
        has_edit_perms = _has_edit_xform_permission(xform, request_user)

        if requires_auth and not has_edit_perms:
            raise PermissionDenied(
                _(u"%(request_user)s is not allowed to make edit submissions "
                  u"to %(form_user)s's %(form_title)s form." % {
                      'request_user': request_user,
                      'form_user': xform.user,
                      'form_title': xform.title}))


def check_submission_permissions(request, xform):
    """Check that permission is required and the request user has permission.

    The user does no have permissions iff:
        * the user is authed,
        * either the profile or the form require auth,
        * the xform user is not submitting.

    Since we have a username, the Instance creation logic will
    handle checking for the forms existence by its id_string.

    :returns: None.
    :raises: PermissionDenied based on the above criteria.
    """
    if request and (xform.user.profile.require_auth or xform.require_auth)\
            and xform.user != request.user\
            and not request.user.has_perm('report_xform', xform):
        raise PermissionDenied(
            _(u"%(request_user)s is not allowed to make submissions "
              u"to %(form_user)s's %(form_title)s form." % {
                  'request_user': request.user,
                  'form_user': xform.user,
                  'form_title': xform.title}))


def save_submission(xform, xml, media_files, new_uuid, submitted_by, status,
                    date_created_override):
    if not date_created_override:
        date_created_override = get_submission_date_from_xml(xml)

    instance = _get_instance(xml, new_uuid, submitted_by, status, xform)

    for f in media_files:
        Attachment.objects.get_or_create(
            instance=instance, media_file=f, mimetype=f.content_type)

    # override date created if required
    if date_created_override:
        if not timezone.is_aware(date_created_override):
            # default to utc?
            date_created_override = timezone.make_aware(
                date_created_override, timezone.utc)
        instance.date_created = date_created_override
        instance.save()

    if instance.xform is not None:
        pi, created = ParsedInstance.objects.get_or_create(
            instance=instance)

    if not created:
        pi.save(async=False)

    return instance


@transaction.commit_manually
def create_instance(username, xml_file, media_files,
                    status=u'submitted_via_web', uuid=None,
                    date_created_override=None, request=None):
    """
    I used to check if this file had been submitted already, I've
    taken this out because it was too slow. Now we're going to create
    a way for an admin to mark duplicate instances. This should
    simplify things a bit.
    Submission cases:
        If there is a username and no uuid, submitting an old ODK form.
        If there is a username and a uuid, submitting a new ODK form.
    """
    try:
        instance = None
        submitted_by = request.user \
            if request and request.user.is_authenticated() else None

        if username:
            username = username.lower()

        xml = xml_file.read()
        xform = get_xform_from_submission(xml, username, uuid)
        check_submission_permissions(request, xform)

        existing_instance_count = Instance.objects.filter(
            xml=xml, xform__user=xform.user).count()

        if existing_instance_count == 0:
            proceed_to_create_instance = True
        else:
            existing_instance = Instance.objects.filter(
                xml=xml, xform__user=xform.user)[0]
            if existing_instance.xform and\
                    not existing_instance.xform.has_start_time:
                proceed_to_create_instance = True
            else:
                # Ignore submission as a duplicate IFF
                #  * a submission's XForm collects start time
                #  * the submitted XML is an exact match with one that
                #    has already been submitted for that user.
                proceed_to_create_instance = False
                raise DuplicateInstance()

        # get new and depracated uuid's
        new_uuid = get_uuid_from_xml(xml)
        duplicate_instances = Instance.objects.filter(uuid=new_uuid)

        if duplicate_instances:
            for f in media_files:
                Attachment.objects.get_or_create(
                    instance=duplicate_instances[0],
                    media_file=f, mimetype=f.content_type)
            # ensure we have saved the extra attachments
            transaction.commit()
            raise DuplicateInstance()

        if proceed_to_create_instance:
            instance = save_submission(xform, xml, media_files, new_uuid,
                                       submitted_by, status,
                                       date_created_override)
            # commit all changes
            transaction.commit()

        return instance
    except Exception:
        transaction.rollback()
        raise


def safe_create_instance(username, xml_file, media_files, uuid, request):
    """Create an instance and catch exceptions.

    :returns: A list [error, instance] where error is None if there was no
        error.
    """
    error = instance = None<commit_msg>PLD: remove proceed_to_create_instance since raise DuplicateInstance handles what it did<commit_after>from datetime import date, datetime
import os
import pytz
import re
import tempfile
import traceback
from xml.dom import Node
from xml.parsers.expat import ExpatError

from django.conf import settings
from django.core.exceptions import ValidationError, PermissionDenied
from django.core.files.storage import get_storage_class
from django.core.mail import mail_admins
from django.core.servers.basehttp import FileWrapper
from django.contrib.auth.models import User
from django.db import IntegrityError, transaction
from django.db.models.signals import pre_delete
from django.http import HttpResponse, HttpResponseNotFound, \
    StreamingHttpResponse
from django.shortcuts import get_object_or_404
from django.utils.encoding import DjangoUnicodeDecodeError
from django.utils.translation import ugettext as _
from django.utils import timezone
from modilabs.utils.subprocess_timeout import ProcessTimedOut
from pyxform.errors import PyXFormError
from pyxform.xform2json import create_survey_element_from_xml
import sys

from onadata.apps.logger.models import Attachment
from onadata.apps.logger.models import Instance
from onadata.apps.logger.models.instance import (
    FormInactiveError,
    InstanceHistory,
    get_id_string_from_xml_str)
from onadata.apps.logger.models import XForm
from onadata.apps.logger.models.xform import XLSFormError
from onadata.apps.logger.xform_instance_parser import (
    InstanceEmptyError,
    InstanceInvalidUserError,
    InstanceMultipleNodeError,
    DuplicateInstance,
    clean_and_parse_xml,
    get_uuid_from_xml,
    get_deprecated_uuid_from_xml,
    get_submission_date_from_xml)
from onadata.apps.viewer.models.data_dictionary import DataDictionary
from onadata.apps.viewer.models.parsed_instance import _remove_from_mongo,\
    xform_instances, ParsedInstance
from onadata.libs.utils import common_tags
from onadata.libs.utils.model_tools import queryset_iterator, set_uuid


OPEN_ROSA_VERSION_HEADER = 'X-OpenRosa-Version'
HTTP_OPEN_ROSA_VERSION_HEADER = 'HTTP_X_OPENROSA_VERSION'
OPEN_ROSA_VERSION = '1.0'
DEFAULT_CONTENT_TYPE = 'text/xml; charset=utf-8'
DEFAULT_CONTENT_LENGTH = settings.DEFAULT_CONTENT_LENGTH

uuid_regex = re.compile(r'<formhub><uuid>([^<]+)</uuid></formhub>',
                        re.DOTALL)

mongo_instances = settings.MONGO_DB.instances


def _get_instance(xml, new_uuid, submitted_by, status, xform):
    # check if its an edit submission
    old_uuid = get_deprecated_uuid_from_xml(xml)
    instances = Instance.objects.filter(uuid=old_uuid)

    if instances:
        # edits
        check_edit_submission_permissions(submitted_by, xform)
        instance = instances[0]
        InstanceHistory.objects.create(
            xml=instance.xml, xform_instance=instance, uuid=old_uuid)
        instance.xml = xml
        instance.uuid = new_uuid
        instance.save()
    else:
        # new submission
        instance = Instance.objects.create(
            xml=xml, user=submitted_by, status=status, xform=xform)

    return instance


def get_uuid_from_submission(xml):
    # parse UUID from uploaded XML
    split_xml = uuid_regex.split(xml)

    # check that xml has UUID
    return len(split_xml) > 1 and split_xml[1] or None


def get_xform_from_submission(xml, username, uuid=None):
        # check alternative form submission ids
        uuid = uuid or get_uuid_from_submission(xml)

        if not username and not uuid:
            raise InstanceInvalidUserError()

        if uuid:
            # try find the form by its uuid which is the ideal condition
            if XForm.objects.filter(uuid=uuid).count() > 0:
                xform = XForm.objects.get(uuid=uuid)
                xform_username = xform.user.username

                if xform_username != username:
                    raise InstanceInvalidUserError()

                return xform

        id_string = get_id_string_from_xml_str(xml)

        return get_object_or_404(XForm, id_string__iexact=id_string,
                                 user__username=username)


def _has_edit_xform_permission(xform, user):
    if isinstance(xform, XForm) and isinstance(user, User):
        return user.has_perm('logger.change_xform', xform)

    return False


def check_edit_submission_permissions(request_user, xform):
    if xform and request_user and request_user.is_authenticated():
        requires_auth = xform.user.profile.require_auth
        has_edit_perms = _has_edit_xform_permission(xform, request_user)

        if requires_auth and not has_edit_perms:
            raise PermissionDenied(
                _(u"%(request_user)s is not allowed to make edit submissions "
                  u"to %(form_user)s's %(form_title)s form." % {
                      'request_user': request_user,
                      'form_user': xform.user,
                      'form_title': xform.title}))


def check_submission_permissions(request, xform):
    """Check that permission is required and the request user has permission.

    The user does no have permissions iff:
        * the user is authed,
        * either the profile or the form require auth,
        * the xform user is not submitting.

    Since we have a username, the Instance creation logic will
    handle checking for the forms existence by its id_string.

    :returns: None.
    :raises: PermissionDenied based on the above criteria.
    """
    if request and (xform.user.profile.require_auth or xform.require_auth)\
            and xform.user != request.user\
            and not request.user.has_perm('report_xform', xform):
        raise PermissionDenied(
            _(u"%(request_user)s is not allowed to make submissions "
              u"to %(form_user)s's %(form_title)s form." % {
                  'request_user': request.user,
                  'form_user': xform.user,
                  'form_title': xform.title}))


def save_submission(xform, xml, media_files, new_uuid, submitted_by, status,
                    date_created_override):
    if not date_created_override:
        date_created_override = get_submission_date_from_xml(xml)

    instance = _get_instance(xml, new_uuid, submitted_by, status, xform)

    for f in media_files:
        Attachment.objects.get_or_create(
            instance=instance, media_file=f, mimetype=f.content_type)

    # override date created if required
    if date_created_override:
        if not timezone.is_aware(date_created_override):
            # default to utc?
            date_created_override = timezone.make_aware(
                date_created_override, timezone.utc)
        instance.date_created = date_created_override
        instance.save()

    if instance.xform is not None:
        pi, created = ParsedInstance.objects.get_or_create(
            instance=instance)

    if not created:
        pi.save(async=False)

    return instance


@transaction.commit_manually
def create_instance(username, xml_file, media_files,
                    status=u'submitted_via_web', uuid=None,
                    date_created_override=None, request=None):
    """
    I used to check if this file had been submitted already, I've
    taken this out because it was too slow. Now we're going to create
    a way for an admin to mark duplicate instances. This should
    simplify things a bit.
    Submission cases:
        If there is a username and no uuid, submitting an old ODK form.
        If there is a username and a uuid, submitting a new ODK form.
    """
    try:
        instance = None
        submitted_by = request.user \
            if request and request.user.is_authenticated() else None

        if username:
            username = username.lower()

        xml = xml_file.read()
        xform = get_xform_from_submission(xml, username, uuid)
        check_submission_permissions(request, xform)

        existing_instance_count = Instance.objects.filter(
            xml=xml, xform__user=xform.user).count()

        if existing_instance_count > 0:
            existing_instance = Instance.objects.filter(
                xml=xml, xform__user=xform.user)[0]
            if not existing_instance.xform or\
                    existing_instance.xform.has_start_time:
                # Ignore submission as a duplicate IFF
                #  * a submission's XForm collects start time
                #  * the submitted XML is an exact match with one that
                #    has already been submitted for that user.
                raise DuplicateInstance()

        # get new and depracated uuid's
        new_uuid = get_uuid_from_xml(xml)
        duplicate_instances = Instance.objects.filter(uuid=new_uuid)

        if duplicate_instances:
            for f in media_files:
                Attachment.objects.get_or_create(
                    instance=duplicate_instances[0],
                    media_file=f, mimetype=f.content_type)
            # ensure we have saved the extra attachments
            transaction.commit()
            raise DuplicateInstance()

        instance = save_submission(xform, xml, media_files, new_uuid,
                                   submitted_by, status, date_created_override)
        # commit all changes
        transaction.commit()

        return instance
    except Exception:
        transaction.rollback()
        raise


def safe_create_instance(username, xml_file, media_files, uuid, request):
    """Create an instance and catch exceptions.

    :returns: A list [error, instance] where error is None if there was no
        error.
    """
    error = instance = None
{'fraction_non_alphanumeric': 0.06671471224132605, 'fraction_numerical': 0.0017502316483063935, 'mean_word_length': 2.7605497483546264, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 88}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228320', 'n_tokens_mistral': 5538, 'n_tokens_neox': 5385, 'n_words': 1454}