<commit_before>from collections import namedtuple
import uuid
from django.test import TestCase
from corehq.apps.userreports.models import DataSourceConfiguration, ReportConfiguration
from corehq.apps.userreports.pillow import get_kafka_ucr_pillow
from corehq.apps.userreports.reports.factory import ReportFactory
from corehq.apps.userreports.tests.utils import doc_to_change
from corehq.apps.userreports.util import get_indicator_adapter


ReportDataTestRow = namedtuple('ReportDataTestRow', ['name', 'number'])


class ReportDataTest(TestCase):

    def setUp(self):
        super(ReportDataTest, self).setUp()
        # Create report
        self.domain = 'test-ucr-report-data'
        self.data_source = DataSourceConfiguration(
            domain=self.domain,
            referenced_doc_type='CommCareCase',
            table_id=uuid.uuid4().hex,
            configured_filter={},
            configured_indicators=[
                {
                    "type": "expression",
                    "expression": {
                        "type": "property_name",
                        "property_name": 'name'
                    },
                    "column_id": 'name',
                    "display_name": 'name',
                    "datatype": "string"
                },
                {
                    "type": "expression",
                    "expression": {
                        "type": "property_name",
                        "property_name": 'number'
                    },
                    "column_id": 'number',
                    "display_name": 'number',
                    "datatype": "integer"
                }
            ],
        )
        self.data_source.validate()
        self.data_source.save()
        get_indicator_adapter(self.data_source).rebuild_table()
        self.addCleanup(self.data_source.delete)

        # initialize a report on the data
        self.report_config = ReportConfiguration(
            domain=self.domain,
            config_id=self.data_source._id,
            aggregation_columns=['doc_id'],
            columns=[
                {
                    "type": "field",
                    "field": "name",
                    "column_id": "name",
                    "display": "Name",
                    "aggregation": "simple",
                },
                {
                    "type": "field",
                    "field": "number",
                    "column_id": "number",
                    "display": "Number",
                    "aggregation": "simple",
                }
            ],
            filters=[],
            configured_charts=[]
        )
        self.report_config.save()
        self.addCleanup(self.report_config.delete)

    def _add_some_rows(self, count):
        rows = [ReportDataTestRow(uuid.uuid4().hex, i) for i in range(count)]
        self._add_rows(rows)
        return rows

    def _add_rows(self, rows):
        pillow = get_kafka_ucr_pillow()
        pillow.bootstrap(configs=[self.data_source])

        def _get_case(row):
            return {
                '_id': uuid.uuid4().hex,
                'domain': self.domain,
                'doc_type': 'CommCareCase',
                'type': 'city',
                'name': row.name,
                'number': row.number,
            }
        for row in rows:
            pillow.process_change(doc_to_change(_get_case(row)))

    def test_basic_query(self):
        # add a few rows to the data source
        rows = self._add_some_rows(3)

        # check the returned data from the report looks right
        report_data_source = ReportFactory.from_spec(self.report_config)
        report_data = report_data_source.get_data()
        self.assertEqual(len(rows), len(report_data))
        rows_by_name = {r.name: r for r in rows}
        for row in report_data:
            self.assertTrue(row['name'] in rows_by_name)
            self.assertEqual(rows_by_name[row['name']].number, row['number'])

    def test_limit(self):
        count = 5
        self._add_some_rows(count)
        report_data_source = ReportFactory.from_spec(self.report_config)
        original_data = report_data_source.get_data()
        self.assertEqual(count, len(original_data))
        limited_data = report_data_source.get_data(limit=3)
        self.assertEqual(3, len(limited_data))
        self.assertEqual(original_data[:3], limited_data)
<commit_msg>failing test imagining how expression columns could work<commit_after>from collections import namedtuple
import uuid
from django.test import TestCase
from corehq.apps.userreports.models import DataSourceConfiguration, ReportConfiguration
from corehq.apps.userreports.pillow import get_kafka_ucr_pillow
from corehq.apps.userreports.reports.factory import ReportFactory
from corehq.apps.userreports.tests.utils import doc_to_change
from corehq.apps.userreports.util import get_indicator_adapter


ReportDataTestRow = namedtuple('ReportDataTestRow', ['name', 'number'])


class ReportDataTest(TestCase):

    def setUp(self):
        super(ReportDataTest, self).setUp()
        # Create report
        self.domain = 'test-ucr-report-data'
        self.data_source = DataSourceConfiguration(
            domain=self.domain,
            referenced_doc_type='CommCareCase',
            table_id=uuid.uuid4().hex,
            configured_filter={},
            configured_indicators=[
                {
                    "type": "expression",
                    "expression": {
                        "type": "property_name",
                        "property_name": 'name'
                    },
                    "column_id": 'name',
                    "display_name": 'name',
                    "datatype": "string"
                },
                {
                    "type": "expression",
                    "expression": {
                        "type": "property_name",
                        "property_name": 'number'
                    },
                    "column_id": 'number',
                    "display_name": 'number',
                    "datatype": "integer"
                }
            ],
        )
        self.data_source.validate()
        self.data_source.save()
        get_indicator_adapter(self.data_source).rebuild_table()
        self.addCleanup(self.data_source.delete)

        # initialize a report on the data
        self.report_config = ReportConfiguration(
            domain=self.domain,
            config_id=self.data_source._id,
            aggregation_columns=['doc_id'],
            columns=[
                {
                    "type": "field",
                    "field": "name",
                    "column_id": "name",
                    "display": "Name",
                    "aggregation": "simple",
                },
                {
                    "type": "field",
                    "field": "number",
                    "column_id": "number",
                    "display": "Number",
                    "aggregation": "simple",
                },
                {
                    "type": "expression",
                    "column_id": "ten",
                    "display": "The Number Ten",
                    "expression": {
                        'type': 'constant',
                        'constant': 10,
                    }
                },
                {
                    "type": "expression",
                    "column_id": "by_tens",
                    "display": "Counting by tens",
                    "expression": {
                        "type": "evaluator",
                        "statement": "a * b",
                        "context_variables": {
                            "a": {
                                "type": "property_name",
                                "property_name": "number",
                            },
                            "b": {
                                "type": "property_name",
                                "property_name": "ten",
                            }
                        }
                    }
                }
            ],
            filters=[],
            configured_charts=[]
        )
        self.report_config.save()
        self.addCleanup(self.report_config.delete)

    def _add_some_rows(self, count):
        rows = [ReportDataTestRow(uuid.uuid4().hex, i) for i in range(count)]
        self._add_rows(rows)
        return rows

    def _add_rows(self, rows):
        pillow = get_kafka_ucr_pillow()
        pillow.bootstrap(configs=[self.data_source])

        def _get_case(row):
            return {
                '_id': uuid.uuid4().hex,
                'domain': self.domain,
                'doc_type': 'CommCareCase',
                'type': 'city',
                'name': row.name,
                'number': row.number,
            }
        for row in rows:
            pillow.process_change(doc_to_change(_get_case(row)))

    def test_basic_query(self):
        # add a few rows to the data source
        rows = self._add_some_rows(3)

        # check the returned data from the report looks right
        report_data_source = ReportFactory.from_spec(self.report_config)
        report_data = report_data_source.get_data()
        self.assertEqual(len(rows), len(report_data))
        rows_by_name = {r.name: r for r in rows}
        for row in report_data:
            self.assertTrue(row['name'] in rows_by_name)
            self.assertEqual(rows_by_name[row['name']].number, row['number'])
            self.assertEqual(10, row['ten'])
            self.assertEqual(10 * row['number'], row['by_tens'])

    def test_limit(self):
        count = 5
        self._add_some_rows(count)
        report_data_source = ReportFactory.from_spec(self.report_config)
        original_data = report_data_source.get_data()
        self.assertEqual(count, len(original_data))
        limited_data = report_data_source.get_data(limit=3)
        self.assertEqual(3, len(limited_data))
        self.assertEqual(original_data[:3], limited_data)

{'fraction_non_alphanumeric': 0.10546214307034125, 'fraction_numerical': 0.002188836931648592, 'mean_word_length': 1.5123719070232442, 'pattern_counts': {'":': 65, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6264750', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2589, 'n_words': 484}