<commit_before>import os
import uuid
from datetime import datetime

import settings
from casexml.apps.case.xform import get_all_extensions_to_close, CaseProcessingResult
from corehq.apps.domain.models import Domain
from corehq.apps.tzmigration import force_phone_timezones_should_be_processed
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.interfaces.processor import FormProcessorInterface, ProcessedForms
from corehq.form_processor.models import XFormInstanceSQL, XFormOperationSQL, XFormAttachmentSQL
from corehq.form_processor.submission_post import CaseStockProcessingResult
from corehq.form_processor.utils import adjust_datetimes
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.utils.general import set_local_domain_sql_backend_override, \
    clear_local_domain_sql_backend_override
from couchforms.models import XFormInstance, doc_types
from fluff.management.commands.ptop_reindexer_fluff import ReindexEventHandler
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider


def do_couch_to_sql_migration(domain):
    # (optional) collect some information about the domain's cases and forms for cross-checking
    set_local_domain_sql_backend_override(domain)
    CouchSqlDomainMigrator(domain).migrate()
    # (optional) compare the information collected to the information at the beginning


class CouchSqlDomainMigrator(object):
    def __init__(self, domain):
        from corehq.apps.tzmigration.planning import DiffDB

        assert should_use_sql_backend(domain)
        self.domain = domain
        db_filepath = get_diff_db_filepath(domain)
        self.diff_db = DiffDB.init(db_filepath)

    def migrate(self):
        self._process_main_forms()
        self._copy_unprocessed_forms()
        self._calculate_case_diffs()

    def _process_main_forms(self):
        last_received_on = datetime.min
        # process main forms (including cases and ledgers)
        for change in _get_main_form_iterator(self.domain).iter_all_changes():
            form = change.get_document()
            wrapped_form = XFormInstance.wrap(form)
            form_received = wrapped_form.received_on
            assert last_received_on <= form_received
            last_received_on = form_received
            print 'processing form {}: {}'.format(form['_id'], form_received)
            self._migrate_form_and_associated_models(wrapped_form)

    def _migrate_form_and_associated_models(self, couch_form):
        from corehq.apps.tzmigration.timezonemigration import json_diff

        sql_form = _migrate_form(self.domain, couch_form)
        _migrate_form_attachments(sql_form, couch_form)
        _migrate_form_operations(sql_form, couch_form)

        diffs = json_diff(couch_form.to_json(), sql_form.to_json())
        self.diff_db.add_diffs(
            'form', couch_form.form_id,
            _filter_form_diffs(couch_form.doc_type, diffs)
        )<commit_msg>add todo<commit_after>import os
import uuid
from datetime import datetime

import settings
from casexml.apps.case.xform import get_all_extensions_to_close, CaseProcessingResult
from corehq.apps.domain.models import Domain
from corehq.apps.tzmigration import force_phone_timezones_should_be_processed
from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.interfaces.processor import FormProcessorInterface, ProcessedForms
from corehq.form_processor.models import XFormInstanceSQL, XFormOperationSQL, XFormAttachmentSQL
from corehq.form_processor.submission_post import CaseStockProcessingResult
from corehq.form_processor.utils import adjust_datetimes
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.utils.general import set_local_domain_sql_backend_override, \
    clear_local_domain_sql_backend_override
from couchforms.models import XFormInstance, doc_types
from fluff.management.commands.ptop_reindexer_fluff import ReindexEventHandler
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider


def do_couch_to_sql_migration(domain):
    # (optional) collect some information about the domain's cases and forms for cross-checking
    set_local_domain_sql_backend_override(domain)
    CouchSqlDomainMigrator(domain).migrate()
    # (optional) compare the information collected to the information at the beginning


class CouchSqlDomainMigrator(object):
    def __init__(self, domain):
        from corehq.apps.tzmigration.planning import DiffDB

        assert should_use_sql_backend(domain)
        self.domain = domain
        db_filepath = get_diff_db_filepath(domain)
        self.diff_db = DiffDB.init(db_filepath)

    def migrate(self):
        self._process_main_forms()
        self._copy_unprocessed_forms()
        self._calculate_case_diffs()
        # TODO: calculate ledger diffs

    def _process_main_forms(self):
        last_received_on = datetime.min
        # process main forms (including cases and ledgers)
        for change in _get_main_form_iterator(self.domain).iter_all_changes():
            form = change.get_document()
            wrapped_form = XFormInstance.wrap(form)
            form_received = wrapped_form.received_on
            assert last_received_on <= form_received
            last_received_on = form_received
            print 'processing form {}: {}'.format(form['_id'], form_received)
            self._migrate_form_and_associated_models(wrapped_form)

    def _migrate_form_and_associated_models(self, couch_form):
        from corehq.apps.tzmigration.timezonemigration import json_diff

        sql_form = _migrate_form(self.domain, couch_form)
        _migrate_form_attachments(sql_form, couch_form)
        _migrate_form_operations(sql_form, couch_form)

        diffs = json_diff(couch_form.to_json(), sql_form.to_json())
        self.diff_db.add_diffs(
            'form', couch_form.form_id,
            _filter_form_diffs(couch_form.doc_type, diffs)
        )
{'fraction_non_alphanumeric': 0.06047865459249677, 'fraction_numerical': 0.0, 'mean_word_length': 4.4782993799822854, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6641861', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1804, 'n_words': 355}