<commit_before>import json
import logging
from datetime import datetime
from functools import partial, wraps
from uuid import uuid4
from xml.sax.saxutils import escape

from django.template.loader import render_to_string

import attr
from memoized import memoized

from casexml.apps.case import const
from casexml.apps.case.sharedmodels import CommCareCaseIndex
from casexml.apps.case.xml.parser import KNOWN_PROPERTIES
from casexml.apps.phone.xml import get_case_xml
from dimagi.utils.parsing import json_format_datetime

from corehq.apps.tzmigration.timezonemigration import MISSING
from corehq.blobs import get_blob_db
from corehq.form_processor.exceptions import CaseNotFound
from corehq.form_processor.models import (
    Attachment,
    XFormInstanceSQL,
    XFormOperationSQL,
)
from corehq.util.metrics import metrics_counter

from .casediff import get_domain
from .casedifftool import format_diffs
from .couchsqlmigration import get_case_and_ledger_updates, save_migrated_models
from .retrydb import get_couch_case

log = logging.getLogger(__name__)


def patch_diffs(doc_diffs, log_cases=False):
    """Patch case diffs

    :param doc_diffs: List of three-tuples as returned by
    `StateDB.iter_doc_diffs()`.
    :returns: A list of case ids to re-diff.
    """
    pending_diffs = []
    dd_count = partial(metrics_counter, tags={"domain": get_domain()})
    for kind, case_id, diffs in doc_diffs:
        assert kind == "CommCareCase", (kind, case_id)
        dd_count("commcare.couchsqlmigration.case.patch")
        try:
            patch_case(case_id, diffs)
        except CannotPatch as err:
            error_diffs = [(kind, case_id, err.diffs)]
            log.warning("cannot patch %s", format_diffs(error_diffs))
            continue
        except Exception:
            error_diffs = [(kind, case_id, [d.json_diff for d in diffs])]
            log.exception("cannot patch %s", format_diffs(error_diffs))
            continue
        pending_diffs.append(case_id)
    return pending_diffs


def patch_case(case_id, diffs):
    case_diffs = [d.json_diff for d in diffs if d.kind != "stock state"]
    try:
        couch_case = get_couch_case(case_id)
    except CaseNotFound:
        raise CannotPatch(case_diffs)
    assert couch_case.domain == get_domain(), (couch_case, get_domain())
    case = PatchCase(couch_case, case_diffs)
    form = PatchForm(case)
    process_patch(form)
    patch_ledgers([d for d in diffs if d.kind == "stock state"])


def patch_ledgers(diffs):
    if diffs:
        # TODO implement ledger patch
        raise CannotPatch([d.json_diff for d in diffs])


def aslist(generator_func):
    @wraps(generator_func)
    def wrapper(*args, **kw):
        return list(generator_func(*args, **kw))
    return wrapper


@attr.s(hash=False)
class PatchCase:
    case = attr.ib()
    diffs = attr.ib()

    def __attrs_post_init__(self):
        self.case_attachments = []
        self._updates = updates = []
        if is_missing_in_sql(self.diffs):<commit_msg>Do not crash case patch on ledger diffs<commit_after>import json
import logging
from datetime import datetime
from functools import partial, wraps
from uuid import uuid4
from xml.sax.saxutils import escape

from django.template.loader import render_to_string

import attr
from memoized import memoized

from casexml.apps.case import const
from casexml.apps.case.sharedmodels import CommCareCaseIndex
from casexml.apps.case.xml.parser import KNOWN_PROPERTIES
from casexml.apps.phone.xml import get_case_xml
from dimagi.utils.parsing import json_format_datetime

from corehq.apps.tzmigration.timezonemigration import MISSING
from corehq.blobs import get_blob_db
from corehq.form_processor.exceptions import CaseNotFound
from corehq.form_processor.models import (
    Attachment,
    XFormInstanceSQL,
    XFormOperationSQL,
)
from corehq.util.metrics import metrics_counter

from .casediff import get_domain
from .casedifftool import format_diffs
from .couchsqlmigration import get_case_and_ledger_updates, save_migrated_models
from .retrydb import get_couch_case

log = logging.getLogger(__name__)


def patch_diffs(doc_diffs, log_cases=False):
    """Patch case diffs

    :param doc_diffs: List of three-tuples as returned by
    `StateDB.iter_doc_diffs()`.
    :returns: A list of case ids to re-diff.
    """
    pending_diffs = []
    dd_count = partial(metrics_counter, tags={"domain": get_domain()})
    for kind, case_id, diffs in doc_diffs:
        assert kind == "CommCareCase", (kind, case_id)
        dd_count("commcare.couchsqlmigration.case.patch")
        try:
            patch_case(case_id, diffs)
        except CannotPatch as err:
            error_diffs = [(kind, case_id, err.diffs)]
            log.warning("cannot patch %s", format_diffs(error_diffs))
            continue
        except Exception:
            error_diffs = [(kind, case_id, [d.json_diff for d in diffs])]
            log.exception("cannot patch %s", format_diffs(error_diffs))
            continue
        pending_diffs.append(case_id)
    return pending_diffs


def patch_case(case_id, diffs):
    case_diffs = [d.json_diff for d in diffs if d.kind != "stock state"]
    if case_diffs:
        try:
            couch_case = get_couch_case(case_id)
        except CaseNotFound:
            raise CannotPatch(case_diffs)
        assert couch_case.domain == get_domain(), (couch_case, get_domain())
        case = PatchCase(couch_case, case_diffs)
        form = PatchForm(case)
        process_patch(form)
    patch_ledgers([d for d in diffs if d.kind == "stock state"])


def patch_ledgers(diffs):
    if diffs:
        # TODO implement ledger patch
        raise CannotPatch([d.json_diff for d in diffs])


def aslist(generator_func):
    @wraps(generator_func)
    def wrapper(*args, **kw):
        return list(generator_func(*args, **kw))
    return wrapper


@attr.s(hash=False)
class PatchCase:
    case = attr.ib()
    diffs = attr.ib()

    def __attrs_post_init__(self):
        self.case_attachments = []
        self._updates = updates = []
        if is_missing_in_sql(self.diffs):
{'fraction_non_alphanumeric': 0.08493510760637424, 'fraction_numerical': 0.0003285690816494168, 'mean_word_length': 3.5365126676602086, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1779078', 'n_tokens_mistral': 1960, 'n_tokens_neox': 1931, 'n_words': 498}