<commit_before>"""Restore logic implementation aiming to minimize database queries

Example case graphs with outcomes:

   a   <--ext-- d(owned) >> a b d
       <--ext-- b

   e(owned)    --ext--> a(closed) >> a b e
               --ext--> b

   b(closed)   <--chi-- a(owned) >> a b c
               <--ext-- c

   a(closed)   <--ext-- d(owned) >> []

   a <--ext-- b <--ext-- c(owned) >> a b c

   a(closed) <--ext-- b <--ext-- c(owned) >> []

   a(closed) <--ext-- b <--ext-- c(owned) <--chi-- d >> []

   a(closed) <--ext-- b <--chi-- c(owned) >> []
"""
import logging
from collections import defaultdict
from itertools import chain, islice

from casexml.apps.case.const import CASE_INDEX_EXTENSION as EXTENSION
from casexml.apps.phone.data_providers.case.load_testing import (
    get_xml_for_response,
)
from casexml.apps.phone.const import ASYNC_RETRY_AFTER
from casexml.apps.phone.data_providers.case.stock import get_stock_payload
from casexml.apps.phone.data_providers.case.utils import get_case_sync_updates
from casexml.apps.phone.tasks import ASYNC_RESTORE_SENT
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors


def do_livequery(timing_context, restore_state, async_task=None):
    """Get case sync restore response

    This function makes no changes to external state other than updating
    the `restore_state.current_sync_log` and progress of `async_task`.
    """
    def index_key(index):
        return '{} {}'.format(index.case_id, index.identifier)

    def is_extension(case_id):
        """Determine if case_id is an extension case

        A case that is both a child and an extension is not an extension.
        """
        return case_id in hosts_by_extension and case_id not in parents_by_child

    def has_live_extension(case_id, cache={}):
        """Check if available case_id has a live extension case

        Do not check for live children because an available parent
        cannot cause it's children to become live. This is unlike an
        available host, which can cause its available extension to
        become live through the recursive rules:

        - A case is available if
            - it is open and not an extension case (applies to host).
            - it is open and is the extension of an available case.
        - A case is live if it is owned and available.

        The result is cached to reduce recursion in subsequent calls
        and to prevent infinite recursion.
        """
        try:
            return cache[case_id]
        except KeyError:
            cache[case_id] = False
        cache[case_id] = result = any(
            ext_id in live_ids      # has live extension
            or ext_id in owned_ids  # ext is owned and available, will be live
            or has_live_extension(ext_id)
            for ext_id in extensions_by_host[case_id]
        )
        return result

    def enliven(case_id):
        """Mark the given case, its extensions and their hosts as live

        This closure mutates `live_ids` from the enclosing function.
        """
        if case_id in live_ids:
            # already live
            return
        debug('enliven(%s)', case_id)
        live_ids.add(case_id)
        # case is open and is the extension of a live case
        ext_ids = extensions_by_host.get(case_id, [])
        # case has live extension
        host_ids = hosts_by_extension.get(case_id, [])
        # case has live child
        parent_ids = parents_by_child.get(case_id, [])
        for cid in chain(ext_ids, host_ids, parent_ids):
            enliven(cid)

    def classify(index, prev_ids):
        """Classify index as either live or extension with live status pending

        This closure mutates case graph data structures from the
        enclosing function.

        :returns: Case id for next related index fetch or IGNORE
        if the related case should be ignored.
        """
        seen_ix.add(index_key(index))
        indices[index.case_id].append(index)
        sub_id = index.case_id
        ref_id = index.referenced_id  # aka parent/host/super
        relationship = index.relationship
        debug("%s --%s--> %s", sub_id, relationship, ref_id)
        if sub_id in live_ids:
            # ref has a live child or extension
            enliven(ref_id)
        elif relationship == EXTENSION:
            if sub_id in open_ids:
                if ref_id in live_ids:
                    # sub is open and is the extension of a live case
                    enliven(sub_id)
                else:
                    # live status pending:
                    # if ref becomes live -> sub is open extension of live case
                    # if sub becomes live -> ref has a live extension
                    extensions_by_host[ref_id].add(sub_id)
                    hosts_by_extension[sub_id].add(ref_id)<commit_msg>Convince myself that livequery doesn't have a hole<commit_after>"""Restore logic implementation aiming to minimize database queries

Example case graphs with outcomes:

   a   <--ext-- d(owned) >> a b d
       <--ext-- b

   e(owned)    --ext--> a(closed) >> a b e
               --ext--> b

   b(closed)   <--chi-- a(owned) >> a b c
               <--ext-- c

   a(closed)   <--ext-- d(owned) >> []

   a <--ext-- b <--ext-- c(owned) >> a b c

   a(closed) <--ext-- b <--ext-- c(owned) >> []

   a(closed) <--ext-- b <--ext-- c(owned) <--chi-- d >> []

   a(closed) <--ext-- b <--chi-- c(owned) >> []
"""
import logging
from collections import defaultdict
from itertools import chain, islice

from casexml.apps.case.const import CASE_INDEX_EXTENSION as EXTENSION
from casexml.apps.phone.data_providers.case.load_testing import (
    get_xml_for_response,
)
from casexml.apps.phone.const import ASYNC_RETRY_AFTER
from casexml.apps.phone.data_providers.case.stock import get_stock_payload
from casexml.apps.phone.data_providers.case.utils import get_case_sync_updates
from casexml.apps.phone.tasks import ASYNC_RESTORE_SENT
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors


def do_livequery(timing_context, restore_state, async_task=None):
    """Get case sync restore response

    This function makes no changes to external state other than updating
    the `restore_state.current_sync_log` and progress of `async_task`.
    """
    def index_key(index):
        return '{} {}'.format(index.case_id, index.identifier)

    def is_extension(case_id):
        """Determine if case_id is an extension case

        A case that is both a child and an extension is not an extension.
        """
        return case_id in hosts_by_extension and case_id not in parents_by_child

    def has_live_extension(case_id, cache={}):
        """Check if available case_id has a live extension case

        Do not check for live children because an available parent
        cannot cause it's children to become live. This is unlike an
        available host, which can cause its available extension to
        become live through the recursive rules:

        - A case is available if
            - it is open and not an extension case (applies to host).
            - it is open and is the extension of an available case.
        - A case is live if it is owned and available.

        The result is cached to reduce recursion in subsequent calls
        and to prevent infinite recursion.
        """
        try:
            return cache[case_id]
        except KeyError:
            cache[case_id] = False
        cache[case_id] = result = any(
            ext_id in live_ids      # has live extension
            or ext_id in owned_ids  # ext is owned and available, will be live
            or has_live_extension(ext_id)
            for ext_id in extensions_by_host[case_id]
        )
        return result

    def enliven(case_id):
        """Mark the given case, its extensions and their hosts as live

        This closure mutates `live_ids` from the enclosing function.
        """
        if case_id in live_ids:
            # already live
            return
        debug('enliven(%s)', case_id)
        live_ids.add(case_id)
        # case is open and is the extension of a live case
        ext_ids = extensions_by_host.get(case_id, [])
        # case has live extension
        host_ids = hosts_by_extension.get(case_id, [])
        # case has live child
        parent_ids = parents_by_child.get(case_id, [])
        for cid in chain(ext_ids, host_ids, parent_ids):
            enliven(cid)

    def classify(index, prev_ids):
        """Classify index as either live or extension with live status pending

        This closure mutates case graph data structures from the
        enclosing function.

        :returns: Case id for next related index fetch or IGNORE
        if the related case should be ignored.
        """
        seen_ix.add(index_key(index))
        indices[index.case_id].append(index)
        sub_id = index.case_id
        ref_id = index.referenced_id  # aka parent/host/super
        relationship = index.relationship
        debug("%s --%s--> %s", sub_id, relationship, ref_id)
        if sub_id in live_ids:
            # ref has a live child or extension
            enliven(ref_id)
            # It does not matter that sub_id -> ref_id never makes it into
            # hosts_by_extension since both are live and therefore this index
            # will not need to be traversed in other liveness calculations.
        elif relationship == EXTENSION:
            if sub_id in open_ids:
                if ref_id in live_ids:
                    # sub is open and is the extension of a live case
                    enliven(sub_id)
                else:
                    # live status pending:
                    # if ref becomes live -> sub is open extension of live case
                    # if sub becomes live -> ref has a live extension
                    extensions_by_host[ref_id].add(sub_id)
                    hosts_by_extension[sub_id].add(ref_id)
{'fraction_non_alphanumeric': 0.08094282848545636, 'fraction_numerical': 0.0, 'mean_word_length': 2.185623003194888, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '18259', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2774, 'n_words': 1012}