<commit_before>from __future__ import absolute_import
import uuid
from datetime import datetime

from contextlib2 import ExitStack
from django.core.files.uploadedfile import UploadedFile
from django.test import TestCase

import settings
from corehq.blobs import get_blob_db
from corehq.blobs.tests.util import TemporaryS3BlobDB, TemporaryFilesystemBlobDB
from corehq.form_processor.backends.sql.dbaccessors import FormAccessorSQL, CaseAccessorSQL
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.exceptions import XFormNotFound, AttachmentNotFound
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.models import (
    XFormInstanceSQL, XFormOperationSQL, XFormAttachmentSQL
)
from corehq.form_processor.parsers.form import apply_deprecation
from corehq.form_processor.tests.utils import (
    create_form_for_test, FormProcessorTestUtils, use_sql_backend
)
from corehq.form_processor.utils import get_simple_form_xml, get_simple_wrapped_form
from corehq.form_processor.utils.xform import TestFormMetadata
from corehq.sql_db.routers import db_for_read_write
from corehq.util.test_utils import trap_extra_setup

DOMAIN = 'test-form-accessor'


@use_sql_backend
class FormAccessorTestsSQL(TestCase):

    def tearDown(self):
        FormProcessorTestUtils.delete_all_sql_forms(DOMAIN)
        FormProcessorTestUtils.delete_all_sql_cases(DOMAIN)
        super(FormAccessorTestsSQL, self).tearDown()

    def test_get_form_by_id(self):
        form = create_form_for_test(DOMAIN)
        with self.assertNumQueries(1, using=form.db):
            form = FormAccessorSQL.get_form(form.form_id)
        self._check_simple_form(form)

    def test_get_form_by_id_missing(self):
        with self.assertRaises(XFormNotFound):
            FormAccessorSQL.get_form('missing_form')

    def test_get_forms(self):
        form1 = create_form_for_test(DOMAIN)
        form2 = create_form_for_test(DOMAIN)

        forms = FormAccessorSQL.get_forms(['missing_form'])
        self.assertEqual(0, len(forms))

        forms = FormAccessorSQL.get_forms([form1.form_id])
        self.assertEqual(1, len(forms))
        self.assertEqual(form1.form_id, forms[0].form_id)

        forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True)
        self.assertEqual(2, len(forms))
        self.assertEqual(form1.form_id, forms[0].form_id)
        self.assertEqual(form2.form_id, forms[1].form_id)

    def test_get_forms_by_last_modified(self):
        start = datetime(2016, 1, 1)
        end = datetime(2018, 1, 1)

        form1 = create_form_for_test(DOMAIN, received_on=datetime(2017, 1, 1))
        create_form_for_test(DOMAIN, received_on=datetime(2015, 1, 1))
        # Test that it gets all states
        form2 = create_form_for_test(
            DOMAIN,
            state=XFormInstanceSQL.ARCHIVED,
            received_on=datetime(2017, 1, 1)
        )
        # Test that other date fields are properly fetched
        form3 = create_form_for_test(
            DOMAIN,
            received_on=datetime(2015, 1, 1),
            edited_on=datetime(2017, 1, 1),
        )

        forms = list(FormAccessorSQL.iter_forms_by_last_modified(start, end))
        self.assertEqual(3, len(forms))
        self.assertEqual(
            {form1.form_id, form2.form_id, form3.form_id},
            {form.form_id for form in forms},
        )

    def test_get_with_attachments(self):
        form = create_form_for_test(DOMAIN)
        with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
            form.get_attachment_meta('form.xml')

        with ExitStack() as stack:
            if settings.USE_PARTITIONED_DATABASE:
                proxy_queries = 1
                stack.enter_context(self.assertNumQueries(1, using=form.db))
            else:
                proxy_queries = 2
            stack.enter_context(self.assertNumQueries(proxy_queries, using=db_for_read_write(XFormAttachmentSQL)))
            form = FormAccessorSQL.get_with_attachments(form.form_id)<commit_msg>update test<commit_after>from __future__ import absolute_import
import uuid
from datetime import datetime

from contextlib2 import ExitStack
from django.core.files.uploadedfile import UploadedFile
from django.test import TestCase

import settings
from corehq.blobs import get_blob_db
from corehq.blobs.tests.util import TemporaryS3BlobDB, TemporaryFilesystemBlobDB
from corehq.form_processor.backends.sql.dbaccessors import FormAccessorSQL, CaseAccessorSQL
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.exceptions import XFormNotFound, AttachmentNotFound
from corehq.form_processor.interfaces.dbaccessors import FormAccessors
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.models import (
    XFormInstanceSQL, XFormOperationSQL, XFormAttachmentSQL
)
from corehq.form_processor.parsers.form import apply_deprecation
from corehq.form_processor.tests.utils import (
    create_form_for_test, FormProcessorTestUtils, use_sql_backend
)
from corehq.form_processor.utils import get_simple_form_xml, get_simple_wrapped_form
from corehq.form_processor.utils.xform import TestFormMetadata
from corehq.sql_db.routers import db_for_read_write
from corehq.util.test_utils import trap_extra_setup

DOMAIN = 'test-form-accessor'


@use_sql_backend
class FormAccessorTestsSQL(TestCase):

    def tearDown(self):
        FormProcessorTestUtils.delete_all_sql_forms(DOMAIN)
        FormProcessorTestUtils.delete_all_sql_cases(DOMAIN)
        super(FormAccessorTestsSQL, self).tearDown()

    def test_get_form_by_id(self):
        form = create_form_for_test(DOMAIN)
        with self.assertNumQueries(1, using=form.db):
            form = FormAccessorSQL.get_form(form.form_id)
        self._check_simple_form(form)

    def test_get_form_by_id_missing(self):
        with self.assertRaises(XFormNotFound):
            FormAccessorSQL.get_form('missing_form')

    def test_get_forms(self):
        form1 = create_form_for_test(DOMAIN)
        form2 = create_form_for_test(DOMAIN)

        forms = FormAccessorSQL.get_forms(['missing_form'])
        self.assertEqual(0, len(forms))

        forms = FormAccessorSQL.get_forms([form1.form_id])
        self.assertEqual(1, len(forms))
        self.assertEqual(form1.form_id, forms[0].form_id)

        forms = FormAccessorSQL.get_forms([form1.form_id, form2.form_id], ordered=True)
        self.assertEqual(2, len(forms))
        self.assertEqual(form1.form_id, forms[0].form_id)
        self.assertEqual(form2.form_id, forms[1].form_id)

    def test_get_forms_by_last_modified(self):
        start = datetime(2016, 1, 1)
        end = datetime(2018, 1, 1)

        form1 = create_form_for_test(DOMAIN, received_on=datetime(2017, 1, 1))
        create_form_for_test(DOMAIN, received_on=datetime(2015, 1, 1))
        # Test that it gets all states
        form2 = create_form_for_test(
            DOMAIN,
            state=XFormInstanceSQL.ARCHIVED,
            received_on=datetime(2017, 1, 1)
        )
        # Test that other date fields are properly fetched
        form3 = create_form_for_test(
            DOMAIN,
            received_on=datetime(2015, 1, 1),
            edited_on=datetime(2017, 1, 1),
        )

        forms = list(FormAccessorSQL.iter_forms_by_last_modified(start, end))
        self.assertEqual(3, len(forms))
        self.assertEqual(
            {form1.form_id, form2.form_id, form3.form_id},
            {form.form_id for form in forms},
        )

    def test_get_with_attachments(self):
        form = create_form_for_test(DOMAIN)
        form = FormAccessorSQL.get_form(form.form_id)  # refetch to clear cached attachments
        with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
            form.get_attachment_meta('form.xml')

        with self.assertNumQueries(1, using=db_for_read_write(XFormAttachmentSQL)):
            form.get_attachment_meta('form.xml')

        with ExitStack() as stack:
            if settings.USE_PARTITIONED_DATABASE:
                proxy_queries = 1
                stack.enter_context(self.assertNumQueries(1, using=form.db))
            else:
                proxy_queries = 2
            stack.enter_context(self.assertNumQueries(proxy_queries, using=db_for_read_write(XFormAttachmentSQL)))
            form = FormAccessorSQL.get_with_attachments(form.form_id)
{'fraction_non_alphanumeric': 0.07933730019834324, 'fraction_numerical': 0.016450822541127057, 'mean_word_length': 3.6285097192224622, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3626395', 'n_tokens_mistral': 2743, 'n_tokens_neox': 2638, 'n_words': 467}