<commit_before>from itertools import groupby
from collections import defaultdict, OrderedDict
from couchdbkit import SchemaListProperty, SchemaProperty, BooleanProperty

from corehq.apps.userreports.expressions.getters import NestedDictGetter
from corehq.apps.app_manager.dbaccessors import get_built_app_ids_for_app_id, get_all_app_ids
from corehq.apps.app_manager.models import Application
from corehq.apps.app_manager.util import get_case_properties
from dimagi.utils.couch.database import iter_docs
from dimagi.ext.couchdbkit import (
    Document,
    DocumentSchema,
    ListProperty,
    StringProperty,
    IntegerProperty,
)
from corehq.apps.export.const import (
    PROPERTY_TAG_UPDATE,
    CASE_HISTORY_PROPERTIES,
    CASE_HISTORY_GROUP_NAME,
)


class ExportItem(DocumentSchema):
    """
    An item for export.
    path is a question path like ["my_group", "q1"] or a case property name
    like ["date_of_birth"].
    """
    path = ListProperty()
    label = StringProperty()
    tag = StringProperty()
    last_occurrence = IntegerProperty()

    @classmethod
    def create_from_question(cls, question, appVersion):
        return cls(
            path=_string_path_to_list(question['value']),
            label=question['label'],
            last_occurrence=appVersion,
        )

    @classmethod
    def merge(cls, one, two):
        item = cls(one.to_json())
        item.last_occurrence = max(one.last_occurrence, two.last_occurrence)
        return item


class ExportColumn(DocumentSchema):
    item = SchemaProperty(ExportItem)
    label = StringProperty()

    def get_value(self, doc, base_path):
        """
        Get the value of self.item of the given doc.
        When base_path is [], doc is a form submission or case,
        when base_path is non empty, doc is a repeat group from a form submission.
        :param doc: A form submission or instance of a repeat group in a submission or case
        :param base_path:
        :return:
        """
        # Confirm the ExportItem's path starts with the base_path
        assert base_path == self.item.path[:len(base_path)]
        # Get the path from the doc root to the desired ExportItem
        path = self.item.path[len(base_path):]
        return NestedDictGetter(path)(doc)


class TableConfiguration(DocumentSchema):
    name = StringProperty()
    repeat_path = ListProperty()
    columns = ListProperty(ExportColumn)

    def get_rows(self, document):
        """
        Return a list of ExportRows generated for the given document.
        :param document: dictionary representation of a form submission or case
        :return: List of ExportRows
        """
        # Note that sub_documents will be [document] if self.repeat_path is []
        sub_documents = self._get_sub_documents(self.repeat_path, [document])
        rows = []
        for doc in sub_documents:

            row_data = []
            for col in self.columns:
                val = col.get_value(doc, self.repeat_path)
                if isinstance(val, list):
                    row_data.extend(val)
                else:
                    row_data.append(val)
            rows.append(ExportRow(data=row_data))
        return rows

    def _get_sub_documents(self, path, docs):
        """
        Return each instance of a repeat group at the path from the given docs.
        If path is [], just return the docs

        >>> TableConfiguration()._get_sub_documents(['foo'], [{'foo': {'bar': 'a'}}, {'foo': {'bar': 'b'}}])
        [{'bar': 'a'}, {'bar': 'b'}]
        >>> TableConfiguration()._get_sub_documents(['foo', 'bar'], [{'foo': [{'bar': {'baz': 'a'}}, {'bar': {'baz': 'b'}},]}]
        [{'baz': 'a'}, {'baz': 'b'}]

        :param path: A list of a strings
        :param docs: A list of dicts representing form submissions
        :return:
        """
        if len(path) == 0:
            return docs

        new_docs = []
        for doc in docs:
            next_doc = doc.get(path[0], {})
            if type(next_doc) == list:
                new_docs.extend(next_doc)
            else:
                new_docs.append(next_doc)
        return self._get_sub_documents(path[1:], new_docs)


class ExportInstance(Document):
    tables = ListProperty(TableConfiguration)

    class Meta:
        app_label = 'export'


class ExportRow(object):
    def __init__(self, data):
        self.data = data


class ScalarItem(ExportItem):
    """
    A text, numeric, date, etc. question or case property
    """


class Option(DocumentSchema):
    """
    This object represents a multiple choice question option.

    last_occurrence is an app build number representing the last version of the app in
    which this option was present.
    """
    last_occurrence = IntegerProperty()
    value = StringProperty()


class MultipleChoiceItem(ExportItem):
    """
    A multiple choice question or case property
    Choices is the union of choices for the question in each of the builds with
    this question.
    """
    options = SchemaListProperty(Option)

    @classmethod
    def create_from_question(cls, question, appVersion):
        item = super(MultipleChoiceItem, cls).create_from_question(question, appVersion)

        for option in question['options']:
            item.options.append(Option(
                last_occurrence=appVersion,
                value=option['value']
            ))
        return item

    @classmethod
    def merge(cls, one, two):
        item = super(MultipleChoiceItem, cls).merge(one, two)
        options = _merge_lists(one.options, two.options,
            keyfn=lambda i: i.value,
            resolvefn=lambda option1, option2:
                Option(
                    value=option1.value,
                    last_occurrence=max(option1.last_occurrence, option2.last_occurrence)
                ),
            copyfn=lambda option: Option(option.to_json())
        )

        item.options = options
        return item


class ExportGroupSchema(DocumentSchema):
    """
    An object representing the `ExportItem`s that would appear in a single export table, such as all the
    questions in a particular repeat group, or all the questions not in any repeat group.
    """
    path = ListProperty()
    items = SchemaListProperty(ExportItem)
    last_occurrence = IntegerProperty()


class ExportDataSchema(DocumentSchema):
    """
    An object representing the things that can be exported for a particular
    form xmlns or case type. It contains a list of ExportGroupSchema.
    """
    group_schemas = SchemaListProperty(ExportGroupSchema)
    datatype_mapping = defaultdict(lambda: ScalarItem, {
        'MSelect': MultipleChoiceItem,
    })

    @staticmethod
    def _merge_schemas(*schemas):
        """Merges two ExportDataSchemas together

        :param schema1: The first ExportDataSchema
        :param schema2: The second ExportDataSchema
        :returns: The merged ExportDataSchema
        """

        schema = ExportDataSchema()

        def resolvefn(group_schema1, group_schema2):
            group_schema = ExportGroupSchema(
                path=group_schema1.path,
                last_occurrence=max(group_schema1.last_occurrence, group_schema2.last_occurrence),
            )
            items = _merge_lists(<commit_msg>DocumentSchema -> Document<commit_after>from itertools import groupby
from collections import defaultdict, OrderedDict
from couchdbkit import SchemaListProperty, SchemaProperty, BooleanProperty

from corehq.apps.userreports.expressions.getters import NestedDictGetter
from corehq.apps.app_manager.dbaccessors import get_built_app_ids_for_app_id, get_all_app_ids
from corehq.apps.app_manager.models import Application
from corehq.apps.app_manager.util import get_case_properties
from dimagi.utils.couch.database import iter_docs
from dimagi.ext.couchdbkit import (
    Document,
    DocumentSchema,
    ListProperty,
    StringProperty,
    IntegerProperty,
)
from corehq.apps.export.const import (
    PROPERTY_TAG_UPDATE,
    CASE_HISTORY_PROPERTIES,
    CASE_HISTORY_GROUP_NAME,
)


class ExportItem(DocumentSchema):
    """
    An item for export.
    path is a question path like ["my_group", "q1"] or a case property name
    like ["date_of_birth"].
    """
    path = ListProperty()
    label = StringProperty()
    tag = StringProperty()
    last_occurrence = IntegerProperty()

    @classmethod
    def create_from_question(cls, question, appVersion):
        return cls(
            path=_string_path_to_list(question['value']),
            label=question['label'],
            last_occurrence=appVersion,
        )

    @classmethod
    def merge(cls, one, two):
        item = cls(one.to_json())
        item.last_occurrence = max(one.last_occurrence, two.last_occurrence)
        return item


class ExportColumn(DocumentSchema):
    item = SchemaProperty(ExportItem)
    label = StringProperty()

    def get_value(self, doc, base_path):
        """
        Get the value of self.item of the given doc.
        When base_path is [], doc is a form submission or case,
        when base_path is non empty, doc is a repeat group from a form submission.
        :param doc: A form submission or instance of a repeat group in a submission or case
        :param base_path:
        :return:
        """
        # Confirm the ExportItem's path starts with the base_path
        assert base_path == self.item.path[:len(base_path)]
        # Get the path from the doc root to the desired ExportItem
        path = self.item.path[len(base_path):]
        return NestedDictGetter(path)(doc)


class TableConfiguration(DocumentSchema):
    name = StringProperty()
    repeat_path = ListProperty()
    columns = ListProperty(ExportColumn)

    def get_rows(self, document):
        """
        Return a list of ExportRows generated for the given document.
        :param document: dictionary representation of a form submission or case
        :return: List of ExportRows
        """
        # Note that sub_documents will be [document] if self.repeat_path is []
        sub_documents = self._get_sub_documents(self.repeat_path, [document])
        rows = []
        for doc in sub_documents:

            row_data = []
            for col in self.columns:
                val = col.get_value(doc, self.repeat_path)
                if isinstance(val, list):
                    row_data.extend(val)
                else:
                    row_data.append(val)
            rows.append(ExportRow(data=row_data))
        return rows

    def _get_sub_documents(self, path, docs):
        """
        Return each instance of a repeat group at the path from the given docs.
        If path is [], just return the docs

        >>> TableConfiguration()._get_sub_documents(['foo'], [{'foo': {'bar': 'a'}}, {'foo': {'bar': 'b'}}])
        [{'bar': 'a'}, {'bar': 'b'}]
        >>> TableConfiguration()._get_sub_documents(['foo', 'bar'], [{'foo': [{'bar': {'baz': 'a'}}, {'bar': {'baz': 'b'}},]}]
        [{'baz': 'a'}, {'baz': 'b'}]

        :param path: A list of a strings
        :param docs: A list of dicts representing form submissions
        :return:
        """
        if len(path) == 0:
            return docs

        new_docs = []
        for doc in docs:
            next_doc = doc.get(path[0], {})
            if type(next_doc) == list:
                new_docs.extend(next_doc)
            else:
                new_docs.append(next_doc)
        return self._get_sub_documents(path[1:], new_docs)


class ExportInstance(Document):
    tables = ListProperty(TableConfiguration)

    class Meta:
        app_label = 'export'


class ExportRow(object):
    def __init__(self, data):
        self.data = data


class ScalarItem(ExportItem):
    """
    A text, numeric, date, etc. question or case property
    """


class Option(DocumentSchema):
    """
    This object represents a multiple choice question option.

    last_occurrence is an app build number representing the last version of the app in
    which this option was present.
    """
    last_occurrence = IntegerProperty()
    value = StringProperty()


class MultipleChoiceItem(ExportItem):
    """
    A multiple choice question or case property
    Choices is the union of choices for the question in each of the builds with
    this question.
    """
    options = SchemaListProperty(Option)

    @classmethod
    def create_from_question(cls, question, appVersion):
        item = super(MultipleChoiceItem, cls).create_from_question(question, appVersion)

        for option in question['options']:
            item.options.append(Option(
                last_occurrence=appVersion,
                value=option['value']
            ))
        return item

    @classmethod
    def merge(cls, one, two):
        item = super(MultipleChoiceItem, cls).merge(one, two)
        options = _merge_lists(one.options, two.options,
            keyfn=lambda i: i.value,
            resolvefn=lambda option1, option2:
                Option(
                    value=option1.value,
                    last_occurrence=max(option1.last_occurrence, option2.last_occurrence)
                ),
            copyfn=lambda option: Option(option.to_json())
        )

        item.options = options
        return item


class ExportGroupSchema(DocumentSchema):
    """
    An object representing the `ExportItem`s that would appear in a single export table, such as all the
    questions in a particular repeat group, or all the questions not in any repeat group.
    """
    path = ListProperty()
    items = SchemaListProperty(ExportItem)
    last_occurrence = IntegerProperty()


class ExportDataSchema(Document):
    """
    An object representing the things that can be exported for a particular
    form xmlns or case type. It contains a list of ExportGroupSchema.
    """
    group_schemas = SchemaListProperty(ExportGroupSchema)
    datatype_mapping = defaultdict(lambda: ScalarItem, {
        'MSelect': MultipleChoiceItem,
    })

    @staticmethod
    def _merge_schemas(*schemas):
        """Merges two ExportDataSchemas together

        :param schema1: The first ExportDataSchema
        :param schema2: The second ExportDataSchema
        :returns: The merged ExportDataSchema
        """

        schema = ExportDataSchema()

        def resolvefn(group_schema1, group_schema2):
            group_schema = ExportGroupSchema(
                path=group_schema1.path,
                last_occurrence=max(group_schema1.last_occurrence, group_schema2.last_occurrence),
            )
            items = _merge_lists(
{'fraction_non_alphanumeric': 0.09074061363168372, 'fraction_numerical': 0.002196444505456792, 'mean_word_length': 2.6207753479125246, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7637210', 'n_tokens_mistral': 4131, 'n_tokens_neox': 3931, 'n_words': 1168}