<commit_before>from collections import namedtuple
import logging

from couchdbkit import ResourceNotFound
from django.db.models import Q
from casexml.apps.case.const import UNOWNED_EXTENSION_OWNER_ID, CASE_INDEX_EXTENSION
from casexml.apps.case.signals import cases_received
from casexml.apps.case.util import validate_phone_datetime
from casexml.apps.phone.cleanliness import should_create_flags_on_submission
from casexml.apps.phone.models import OwnershipCleanlinessFlag
from corehq.toggles import LOOSE_SYNC_TOKEN_VALIDATION, EXTENSION_CASES_SYNC_ENABLED
from corehq.apps.users.util import SYSTEM_USER_ID
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from couchforms.models import XFormInstance
from casexml.apps.case.exceptions import (
    NoDomainProvided,
)
from django.conf import settings

from casexml.apps.case import const
from casexml.apps.case.xml.parser import case_update_from_block
from dimagi.utils.logging import notify_exception


# Lightweight class used to store the dirtyness of a case/owner pair.
DirtinessFlag = namedtuple('DirtinessFlag', ['case_id', 'owner_id'])


class CaseProcessingResult(object):
    """
    Lightweight class used to collect results of case processing
    """

    def __init__(self, domain, cases, dirtiness_flags, extensions_to_close=None):
        self.domain = domain
        self.cases = cases
        self.dirtiness_flags = dirtiness_flags
        if extensions_to_close is None:
            extensions_to_close = set()
        self.extensions_to_close = extensions_to_close

    def get_clean_owner_ids(self):
        dirty_flags = self.get_flags_to_save()
        return {c.owner_id for c in self.cases if c.owner_id and c.owner_id not in dirty_flags}

    def set_cases(self, cases):
        self.cases = cases

    def get_flags_to_save(self):
        return {f.owner_id: f.case_id for f in self.dirtiness_flags}

    def close_extensions(self):
        from casexml.apps.case.cleanup import close_cases
        extensions_to_close = list(self.extensions_to_close)
        if extensions_to_close:
            return close_cases(list(self.extensions_to_close), self.domain, SYSTEM_USER_ID)

    def commit_dirtiness_flags(self):
        """
        Updates any dirtiness flags in the database.
        """
        if self.domain:
            flags_to_save = self.get_flags_to_save()
            if should_create_flags_on_submission(self.domain):
                assert settings.UNIT_TESTING  # this is currently only true when unit testing
                all_touched_ids = set(flags_to_save.keys()) | self.get_clean_owner_ids()
                to_update = {f.owner_id: f for f in OwnershipCleanlinessFlag.objects.filter(
                    domain=self.domain,
                    owner_id__in=list(all_touched_ids),
                )}
                for owner_id in all_touched_ids:
                    if owner_id not in to_update:
                        # making from scratch - default to clean, but set to dirty if needed
                        flag = OwnershipCleanlinessFlag(domain=self.domain, owner_id=owner_id, is_clean=True)
                        if owner_id in flags_to_save:
                            flag.is_clean = False
                            flag.hint = flags_to_save[owner_id]
                        flag.save()
                    else:
                        # updating - only save if we are marking dirty or setting a hint
                        flag = to_update[owner_id]
                        if owner_id in flags_to_save and (flag.is_clean or not flag.hint):
                            flag.is_clean = False
                            flag.hint = flags_to_save[owner_id]
                            flag.save()
            else:
                # only update the flags that are already in the database
                flags_to_update = OwnershipCleanlinessFlag.objects.filter(
                    Q(domain=self.domain),
                    Q(owner_id__in=flags_to_save.keys()),
                    Q(is_clean=True) | Q(hint__isnull=True)
                )
                for flag in flags_to_update:
                    flag.is_clean = False
                    flag.hint = flags_to_save[flag.owner_id]
                    flag.save()


def process_cases_with_casedb(xforms, case_db, config=None):
    config = config or CaseProcessingConfig()
    case_processing_result = _get_or_update_cases(xforms, case_db)
    cases = case_processing_result.cases
    xform = xforms[0]

    # handle updating the sync records for apps that use sync mode
    try:
        relevant_log = xform.get_sync_token()
    except ResourceNotFound:
        if LOOSE_SYNC_TOKEN_VALIDATION.enabled(xform.domain):
            relevant_log = None
        else:
            raise

    if relevant_log:
        # in reconciliation mode, things can be unexpected
        relevant_log.strict = config.strict_asserts
        from casexml.apps.case.util import update_sync_log_with_checks
        update_sync_log_with_checks(relevant_log, xform, cases, case_db,
                                    case_id_blacklist=config.case_id_blacklist)

    try:
        cases_received.send(sender=None, xform=xform, cases=cases)
    except Exception as e:
        # don't let the exceptions in signals prevent standard case processing
        notify_exception(
            None,
            'something went wrong sending the cases_received signal '
            'for form %s: %s' % (xform.form_id, e)
        )

    for case in cases:
        case_db.post_process_case(case, xform)
        case_db.mark_changed(case)

    case_processing_result.set_cases(cases)
    return case_processing_result


class CaseProcessingConfig(object):

    def __init__(self, strict_asserts=True, case_id_blacklist=None):
        self.strict_asserts = strict_asserts
        self.case_id_blacklist = case_id_blacklist if case_id_blacklist is not None else []

    def __repr__(self):
        return 'strict: {strict}, ids: {ids}'.format(
            strict=self.strict_asserts,
            ids=", ".join(self.case_id_blacklist)
        )


def get_and_check_xform_domain(xform):
    try:
        domain = xform.domain
    except AttributeError:
        domain = None

    if not domain and settings.CASEXML_FORCE_DOMAIN_CHECK:
        raise NoDomainProvided()

    return domain


def _get_or_update_cases(xforms, case_db):
    """
    Given an xform document, update any case blocks found within it,
    returning a dictionary mapping the case ids affected to the
    couch case document objects
    """
    domain = getattr(case_db, 'domain', None)
    touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms)

    # once we've gotten through everything, validate all indices
    # and check for new dirtiness flags
    def _validate_indices(case):
        if case.indices:
            for index in case.indices:
                # call get and not doc_exists to force domain checking
                # see CaseDbCache._validate_case
                referenced_case = case_db.get(index.referenced_id)
                if not referenced_case:
                    # just log, don't raise an error or modify the index
                    logging.error(
                        "Case '%s' references non-existent case '%s'",
                        case.case_id,
                        index.referenced_id,
                    )

    def _get_dirtiness_flags_for_outgoing_indices(case, tree_owners=None):
        """ if the outgoing indices touch cases owned by another user this cases owner is dirty """
        if tree_owners is None:
            tree_owners = set()

        extension_indices = [index for index in case.indices if index.relationship == CASE_INDEX_EXTENSION]

        unowned_host_cases = []
        for index in extension_indices:
            host_case = case_db.get(index.referenced_id)
            if (
                host_case
                and host_case.owner_id == UNOWNED_EXTENSION_OWNER_ID
                and host_case not in unowned_host_cases
            ):
                unowned_host_cases.append(host_case)

        owner_ids = {case_db.get(index.referenced_id).owner_id
                     for index in case.indices if case_db.get(index.referenced_id)} | tree_owners
        potential_clean_owner_ids = owner_ids | set([UNOWNED_EXTENSION_OWNER_ID])
        more_than_one_owner_touched = len(owner_ids) > 1
        touches_different_owner = len(owner_ids) == 1 and case.owner_id not in potential_clean_owner_ids

        if (more_than_one_owner_touched or touches_different_owner):
            yield DirtinessFlag(case.case_id, case.owner_id)
            if extension_indices:
                # If this case is an extension, each of the touched cases is also dirty
                for index in case.indices:
                    referenced_case = case_db.get(index.referenced_id)
                    yield DirtinessFlag(referenced_case.case_id, referenced_case.owner_id)

        if case.owner_id != UNOWNED_EXTENSION_OWNER_ID:
            tree_owners.add(case.owner_id)
        for unowned_host_case in unowned_host_cases:
            # A host case of this extension is unowned, which means it could potentially touch an owned case
            # Check these unowned cases' outgoing indices and mark dirty if appropriate
            for dirtiness_flag in _get_dirtiness_flags_for_outgoing_indices(unowned_host_case,
                                                                            tree_owners=tree_owners):
                yield dirtiness_flag

    def _get_dirtiness_flags_for_child_cases(cases):
        child_cases = case_db.get_reverse_indexed_cases([c.case_id for c in cases])
        case_owner_map = dict((case.case_id, case.owner_id) for case in cases)
        for child_case in child_cases:
            for index in child_case.indices:
                if (index.referenced_id in case_owner_map
                        and child_case.owner_id != case_owner_map[index.referenced_id]):
                    yield DirtinessFlag(child_case.case_id, child_case.owner_id)

    def _get_dirtiness_flags_for_reassigned_case(case_metas):
        # for reassigned cases, we mark them temporarily dirty to allow phones to sync
        # the latest changes. these will get cleaned up when the weekly rebuild triggers
        for case_update_meta in case_metas:
            if _is_change_of_ownership(case_update_meta.previous_owner_id, case_update_meta.case.owner_id):
                yield DirtinessFlag(case_update_meta.case.case_id, case_update_meta.previous_owner_id)

    dirtiness_flags = []
    extensions_to_close = set()

    # process the temporary dirtiness flags first so that any hints for real dirtiness get overridden
    dirtiness_flags += list(_get_dirtiness_flags_for_reassigned_case(touched_cases.values()))
    for case_update_meta in touched_cases.values():
        _validate_indices(case_update_meta.case)
        extensions_to_close = extensions_to_close | get_extensions_to_close(case_update_meta.case, domain)
        dirtiness_flags += list(_get_dirtiness_flags_for_outgoing_indices(case_update_meta.case))
    dirtiness_flags += list(_get_dirtiness_flags_for_child_cases([meta.case for meta in touched_cases.values()]))

    return CaseProcessingResult(
        domain,
        [update.case for update in touched_cases.values()],
        dirtiness_flags,
        extensions_to_close
    )


def _is_change_of_ownership(previous_owner_id, next_owner_id):
    return (
        previous_owner_id
        and previous_owner_id != UNOWNED_EXTENSION_OWNER_ID
        and previous_owner_id != next_owner_id
    )


def get_extensions_to_close(case, domain):
    outgoing_extension_indices = [index.relationship for index in case.indices
                                  if index.relationship == CASE_INDEX_EXTENSION]
    if not outgoing_extension_indices and case.closed and EXTENSION_CASES_SYNC_ENABLED.enabled(domain):
        return get_extension_chain([case.case_id], domain)<commit_msg>Allow SQL domains to close extensions<commit_after>from collections import namedtuple
import logging

from couchdbkit import ResourceNotFound
from django.db.models import Q
from casexml.apps.case.const import UNOWNED_EXTENSION_OWNER_ID, CASE_INDEX_EXTENSION
from casexml.apps.case.signals import cases_received
from casexml.apps.case.util import validate_phone_datetime
from casexml.apps.phone.cleanliness import should_create_flags_on_submission
from casexml.apps.phone.models import OwnershipCleanlinessFlag
from corehq.toggles import LOOSE_SYNC_TOKEN_VALIDATION, EXTENSION_CASES_SYNC_ENABLED
from corehq.apps.users.util import SYSTEM_USER_ID
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from couchforms.models import XFormInstance
from casexml.apps.case.exceptions import (
    NoDomainProvided,
)
from django.conf import settings

from casexml.apps.case import const
from casexml.apps.case.xml.parser import case_update_from_block
from dimagi.utils.logging import notify_exception


# Lightweight class used to store the dirtyness of a case/owner pair.
DirtinessFlag = namedtuple('DirtinessFlag', ['case_id', 'owner_id'])


class CaseProcessingResult(object):
    """
    Lightweight class used to collect results of case processing
    """

    def __init__(self, domain, cases, dirtiness_flags, extensions_to_close=None):
        self.domain = domain
        self.cases = cases
        self.dirtiness_flags = dirtiness_flags
        if extensions_to_close is None:
            extensions_to_close = set()
        self.extensions_to_close = extensions_to_close

    def get_clean_owner_ids(self):
        dirty_flags = self.get_flags_to_save()
        return {c.owner_id for c in self.cases if c.owner_id and c.owner_id not in dirty_flags}

    def set_cases(self, cases):
        self.cases = cases

    def get_flags_to_save(self):
        return {f.owner_id: f.case_id for f in self.dirtiness_flags}

    def close_extensions(self):
        from casexml.apps.case.cleanup import close_cases
        extensions_to_close = list(self.extensions_to_close)
        if extensions_to_close:
            return close_cases(list(self.extensions_to_close), self.domain, SYSTEM_USER_ID)

    def commit_dirtiness_flags(self):
        """
        Updates any dirtiness flags in the database.
        """
        if self.domain:
            flags_to_save = self.get_flags_to_save()
            if should_create_flags_on_submission(self.domain):
                assert settings.UNIT_TESTING  # this is currently only true when unit testing
                all_touched_ids = set(flags_to_save.keys()) | self.get_clean_owner_ids()
                to_update = {f.owner_id: f for f in OwnershipCleanlinessFlag.objects.filter(
                    domain=self.domain,
                    owner_id__in=list(all_touched_ids),
                )}
                for owner_id in all_touched_ids:
                    if owner_id not in to_update:
                        # making from scratch - default to clean, but set to dirty if needed
                        flag = OwnershipCleanlinessFlag(domain=self.domain, owner_id=owner_id, is_clean=True)
                        if owner_id in flags_to_save:
                            flag.is_clean = False
                            flag.hint = flags_to_save[owner_id]
                        flag.save()
                    else:
                        # updating - only save if we are marking dirty or setting a hint
                        flag = to_update[owner_id]
                        if owner_id in flags_to_save and (flag.is_clean or not flag.hint):
                            flag.is_clean = False
                            flag.hint = flags_to_save[owner_id]
                            flag.save()
            else:
                # only update the flags that are already in the database
                flags_to_update = OwnershipCleanlinessFlag.objects.filter(
                    Q(domain=self.domain),
                    Q(owner_id__in=flags_to_save.keys()),
                    Q(is_clean=True) | Q(hint__isnull=True)
                )
                for flag in flags_to_update:
                    flag.is_clean = False
                    flag.hint = flags_to_save[flag.owner_id]
                    flag.save()


def process_cases_with_casedb(xforms, case_db, config=None):
    config = config or CaseProcessingConfig()
    case_processing_result = _get_or_update_cases(xforms, case_db)
    cases = case_processing_result.cases
    xform = xforms[0]

    # handle updating the sync records for apps that use sync mode
    try:
        relevant_log = xform.get_sync_token()
    except ResourceNotFound:
        if LOOSE_SYNC_TOKEN_VALIDATION.enabled(xform.domain):
            relevant_log = None
        else:
            raise

    if relevant_log:
        # in reconciliation mode, things can be unexpected
        relevant_log.strict = config.strict_asserts
        from casexml.apps.case.util import update_sync_log_with_checks
        update_sync_log_with_checks(relevant_log, xform, cases, case_db,
                                    case_id_blacklist=config.case_id_blacklist)

    try:
        cases_received.send(sender=None, xform=xform, cases=cases)
    except Exception as e:
        # don't let the exceptions in signals prevent standard case processing
        notify_exception(
            None,
            'something went wrong sending the cases_received signal '
            'for form %s: %s' % (xform.form_id, e)
        )

    for case in cases:
        case_db.post_process_case(case, xform)
        case_db.mark_changed(case)

    case_processing_result.set_cases(cases)
    return case_processing_result


class CaseProcessingConfig(object):

    def __init__(self, strict_asserts=True, case_id_blacklist=None):
        self.strict_asserts = strict_asserts
        self.case_id_blacklist = case_id_blacklist if case_id_blacklist is not None else []

    def __repr__(self):
        return 'strict: {strict}, ids: {ids}'.format(
            strict=self.strict_asserts,
            ids=", ".join(self.case_id_blacklist)
        )


def get_and_check_xform_domain(xform):
    try:
        domain = xform.domain
    except AttributeError:
        domain = None

    if not domain and settings.CASEXML_FORCE_DOMAIN_CHECK:
        raise NoDomainProvided()

    return domain


def _get_or_update_cases(xforms, case_db):
    """
    Given an xform document, update any case blocks found within it,
    returning a dictionary mapping the case ids affected to the
    couch case document objects
    """
    domain = getattr(case_db, 'domain', None)
    touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms)

    # once we've gotten through everything, validate all indices
    # and check for new dirtiness flags
    def _validate_indices(case):
        if case.indices:
            for index in case.indices:
                # call get and not doc_exists to force domain checking
                # see CaseDbCache._validate_case
                referenced_case = case_db.get(index.referenced_id)
                if not referenced_case:
                    # just log, don't raise an error or modify the index
                    logging.error(
                        "Case '%s' references non-existent case '%s'",
                        case.case_id,
                        index.referenced_id,
                    )

    def _get_dirtiness_flags_for_outgoing_indices(case, tree_owners=None):
        """ if the outgoing indices touch cases owned by another user this cases owner is dirty """
        if tree_owners is None:
            tree_owners = set()

        extension_indices = [index for index in case.indices if index.relationship == CASE_INDEX_EXTENSION]

        unowned_host_cases = []
        for index in extension_indices:
            host_case = case_db.get(index.referenced_id)
            if (
                host_case
                and host_case.owner_id == UNOWNED_EXTENSION_OWNER_ID
                and host_case not in unowned_host_cases
            ):
                unowned_host_cases.append(host_case)

        owner_ids = {case_db.get(index.referenced_id).owner_id
                     for index in case.indices if case_db.get(index.referenced_id)} | tree_owners
        potential_clean_owner_ids = owner_ids | set([UNOWNED_EXTENSION_OWNER_ID])
        more_than_one_owner_touched = len(owner_ids) > 1
        touches_different_owner = len(owner_ids) == 1 and case.owner_id not in potential_clean_owner_ids

        if (more_than_one_owner_touched or touches_different_owner):
            yield DirtinessFlag(case.case_id, case.owner_id)
            if extension_indices:
                # If this case is an extension, each of the touched cases is also dirty
                for index in case.indices:
                    referenced_case = case_db.get(index.referenced_id)
                    yield DirtinessFlag(referenced_case.case_id, referenced_case.owner_id)

        if case.owner_id != UNOWNED_EXTENSION_OWNER_ID:
            tree_owners.add(case.owner_id)
        for unowned_host_case in unowned_host_cases:
            # A host case of this extension is unowned, which means it could potentially touch an owned case
            # Check these unowned cases' outgoing indices and mark dirty if appropriate
            for dirtiness_flag in _get_dirtiness_flags_for_outgoing_indices(unowned_host_case,
                                                                            tree_owners=tree_owners):
                yield dirtiness_flag

    def _get_dirtiness_flags_for_child_cases(cases):
        child_cases = case_db.get_reverse_indexed_cases([c.case_id for c in cases])
        case_owner_map = dict((case.case_id, case.owner_id) for case in cases)
        for child_case in child_cases:
            for index in child_case.indices:
                if (index.referenced_id in case_owner_map
                        and child_case.owner_id != case_owner_map[index.referenced_id]):
                    yield DirtinessFlag(child_case.case_id, child_case.owner_id)

    def _get_dirtiness_flags_for_reassigned_case(case_metas):
        # for reassigned cases, we mark them temporarily dirty to allow phones to sync
        # the latest changes. these will get cleaned up when the weekly rebuild triggers
        for case_update_meta in case_metas:
            if _is_change_of_ownership(case_update_meta.previous_owner_id, case_update_meta.case.owner_id):
                yield DirtinessFlag(case_update_meta.case.case_id, case_update_meta.previous_owner_id)

    dirtiness_flags = []
    extensions_to_close = set()

    # process the temporary dirtiness flags first so that any hints for real dirtiness get overridden
    dirtiness_flags += list(_get_dirtiness_flags_for_reassigned_case(touched_cases.values()))
    for case_update_meta in touched_cases.values():
        _validate_indices(case_update_meta.case)
        extensions_to_close = extensions_to_close | get_extensions_to_close(case_update_meta.case, domain)
        dirtiness_flags += list(_get_dirtiness_flags_for_outgoing_indices(case_update_meta.case))
    dirtiness_flags += list(_get_dirtiness_flags_for_child_cases([meta.case for meta in touched_cases.values()]))

    return CaseProcessingResult(
        domain,
        [update.case for update in touched_cases.values()],
        dirtiness_flags,
        extensions_to_close
    )


def _is_change_of_ownership(previous_owner_id, next_owner_id):
    return (
        previous_owner_id
        and previous_owner_id != UNOWNED_EXTENSION_OWNER_ID
        and previous_owner_id != next_owner_id
    )


def get_extensions_to_close(case, domain):
    outgoing_extension_indices = [index.relationship for index in case.indices
                                  if index.relationship == CASE_INDEX_EXTENSION]
    if not outgoing_extension_indices and case.closed and EXTENSION_CASES_SYNC_ENABLED.enabled(domain):
        return CaseAccessors(domain).get_extension_chain([case.case_id])
{'fraction_non_alphanumeric': 0.0607474705760892, 'fraction_numerical': 0.00024778030146603347, 'mean_word_length': 2.5070238957277335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3010068', 'n_tokens_mistral': 6956, 'n_tokens_neox': 6720, 'n_words': 1657}