<commit_before>from datetime import datetime, timedelta
import os

from mock import patch
from nose.tools import raises

from onadata.apps.odk_logger.models.instance import Instance
from onadata.apps.main.tests.test_base import TestBase
from onadata.libs.data.query import get_form_submissions_grouped_by_field


class TestTools(TestBase):

    def setUp(self):
        super(self.__class__, self).setUp()
        self._create_user_and_login()
        self._publish_transportation_form()

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_grouped_by_field(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()

        count_key = 'count'
        fields = ['_submission_time', '_xform_id_string']

        count = len(self.xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

    @patch('onadata.apps.odk_logger.models.instance.submission_time')
    def test_get_form_submissions_grouped_by_field_datetime_to_date(
            self, mock_time):
        now = datetime(2014, 01, 01)
        times = [now, now + timedelta(seconds=1), now + timedelta(seconds=2),
                 now + timedelta(seconds=3)]
        mock_time.side_effect = times
        self._make_submissions()

        count_key = 'count'
        fields = ['_submission_time']

        count = len(self.xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[field], str(now.date()))
            self.assertEqual(result[count_key], count)

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_two_xforms(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()
        self._publish_xls_file(os.path.join(
            "fixtures",
            "gps", "gps.xls"))

        first_xform = self.xform
        self.xform = self.user.xforms.all().order_by('-pk')[0]

        self._make_submission(os.path.join(
            'onadata', 'apps', 'main', 'tests', 'fixtures', 'gps',
            'instances', 'gps_1980-01-23_20-52-08.xml'))

        count_key = 'count'
        fields = ['_submission_time', '_xform_id_string']

        count = len(self.xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

        count = len(first_xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                first_xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_xform_no_submissions(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()
        self._publish_xls_file(os.path.join(
            "fixtures",
            "gps", "gps.xls"))

        self.xform = self.user.xforms.all().order_by('-pk')[0]

        fields = ['_submission_time', '_xform_id_string']

        count = len(self.xform.surveys.all())
        self.assertEqual(count, 0)
        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)
            self.assertEqual(result, [])

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_grouped_by_field_sets_name(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()

        count_key = 'count'
        fields = ['_submission_time', '_xform_id_string']
        name = '_my_name'

        xform = self.user.xforms.all()[0]
        count = len(xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                xform, field, name)[0]

            self.assertEqual([name, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

    @raises(ValueError)
    def test_get_form_submissions_grouped_by_field_bad_field(self):
        self._make_submissions()

        field = '_bad_field'
        xform = self.user.xforms.all()[0]

        get_form_submissions_grouped_by_field(xform, field)

    # TODO: should fix query logic so value is returned as 1, in which case the
    # test should check that the count is 1 and not stripped
    def test_get_form_submissions_when_response_not_provided(self):
        """
        Test that the None value is stripped when of the submissions
        doesnt have a response for the specified field
        """
        self._make_submissions()

        count = Instance.objects.count()

        # make submission that doesnt have a response for
        # `available_transportation_types_to_referral_facility`
        path = os.path.join(
            self.this_directory, 'fixtures', 'transportation',
            'instances', 'transport_no_response', 'transport_no_response.xml')
        self._make_submission(path, self.user.username)
        self.assertEqual(Instance.objects.count(), count + 1)

        field = 'transport/available_transportation_types_to_referral_facility'
        xform = self.user.xforms.all()[0]

        results = get_form_submissions_grouped_by_field(<commit_msg>fixed todo descr. on count test<commit_after>from datetime import datetime, timedelta
import os

from mock import patch
from nose.tools import raises

from onadata.apps.odk_logger.models.instance import Instance
from onadata.apps.main.tests.test_base import TestBase
from onadata.libs.data.query import get_form_submissions_grouped_by_field


class TestTools(TestBase):

    def setUp(self):
        super(self.__class__, self).setUp()
        self._create_user_and_login()
        self._publish_transportation_form()

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_grouped_by_field(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()

        count_key = 'count'
        fields = ['_submission_time', '_xform_id_string']

        count = len(self.xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

    @patch('onadata.apps.odk_logger.models.instance.submission_time')
    def test_get_form_submissions_grouped_by_field_datetime_to_date(
            self, mock_time):
        now = datetime(2014, 01, 01)
        times = [now, now + timedelta(seconds=1), now + timedelta(seconds=2),
                 now + timedelta(seconds=3)]
        mock_time.side_effect = times
        self._make_submissions()

        count_key = 'count'
        fields = ['_submission_time']

        count = len(self.xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[field], str(now.date()))
            self.assertEqual(result[count_key], count)

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_two_xforms(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()
        self._publish_xls_file(os.path.join(
            "fixtures",
            "gps", "gps.xls"))

        first_xform = self.xform
        self.xform = self.user.xforms.all().order_by('-pk')[0]

        self._make_submission(os.path.join(
            'onadata', 'apps', 'main', 'tests', 'fixtures', 'gps',
            'instances', 'gps_1980-01-23_20-52-08.xml'))

        count_key = 'count'
        fields = ['_submission_time', '_xform_id_string']

        count = len(self.xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

        count = len(first_xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                first_xform, field)[0]

            self.assertEqual([field, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_xform_no_submissions(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()
        self._publish_xls_file(os.path.join(
            "fixtures",
            "gps", "gps.xls"))

        self.xform = self.user.xforms.all().order_by('-pk')[0]

        fields = ['_submission_time', '_xform_id_string']

        count = len(self.xform.surveys.all())
        self.assertEqual(count, 0)
        for field in fields:
            result = get_form_submissions_grouped_by_field(
                self.xform, field)
            self.assertEqual(result, [])

    @patch('django.utils.timezone.now')
    def test_get_form_submissions_grouped_by_field_sets_name(self, mock_time):
        mock_time.return_value = datetime.now()
        self._make_submissions()

        count_key = 'count'
        fields = ['_submission_time', '_xform_id_string']
        name = '_my_name'

        xform = self.user.xforms.all()[0]
        count = len(xform.surveys.all())

        for field in fields:
            result = get_form_submissions_grouped_by_field(
                xform, field, name)[0]

            self.assertEqual([name, count_key], sorted(result.keys()))
            self.assertEqual(result[count_key], count)

    @raises(ValueError)
    def test_get_form_submissions_grouped_by_field_bad_field(self):
        self._make_submissions()

        field = '_bad_field'
        xform = self.user.xforms.all()[0]

        get_form_submissions_grouped_by_field(xform, field)

    # TODO: test will pass when count query is fixed to return actual count of
    # missing/None values, it currently always returns 0 for None i.e. no responses
    def test_get_form_submissions_when_response_not_provided(self):
        """
        Test that the None value is stripped when of the submissions
        doesnt have a response for the specified field
        """
        self._make_submissions()

        count = Instance.objects.count()

        # make submission that doesnt have a response for
        # `available_transportation_types_to_referral_facility`
        path = os.path.join(
            self.this_directory, 'fixtures', 'transportation',
            'instances', 'transport_no_response', 'transport_no_response.xml')
        self._make_submission(path, self.user.username)
        self.assertEqual(Instance.objects.count(), count + 1)

        field = 'transport/available_transportation_types_to_referral_facility'
        xform = self.user.xforms.all()[0]

        results = get_form_submissions_grouped_by_field(
{'fraction_non_alphanumeric': 0.10705638187478381, 'fraction_numerical': 0.006658595641646489, 'mean_word_length': 2.7258376288659796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5509921', 'n_tokens_mistral': 3674, 'n_tokens_neox': 3485, 'n_words': 626}