<commit_before>import logging
import sys
import traceback

from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
from django.db import transaction

from corehq.dbaccessors.couchapps.all_docs import get_all_docs_with_doc_types, get_doc_count_by_type
from corehq.util.couchdb_management import couch_config
from corehq.util.django_migrations import skip_on_fresh_install

logger = logging.getLogger(__name__)


class PopulateSQLCommand(BaseCommand):
    """
        Base class for migrating couch docs to sql models.

        Adds a SQL object for any couch doc that doesn't yet have one.
        Override all methods that raise NotImplementedError and, optionoally, couch_db_slug.
    """
    AUTO_MIGRATE_ITEMS_LIMIT = 1000

    @classmethod
    def couch_db_slug(cls):
        # Override this if couch model was not stored in the main commcarehq database
        return None

    @classmethod
    def couch_doc_type(cls):
        raise NotImplementedError()

    @classmethod
    def sql_class(self):
        raise NotImplementedError()

    def update_or_create_sql_object(self, doc):
        """
        This should find and update the sql object that corresponds to the given doc,
        or create it if it doesn't yet exist. This method is responsible for saving
        the sql object.
        """
        raise NotImplementedError()

    @classmethod
    def diff_couch_and_sql(cls, couch, sql):
        """
        This should compare each attribute of the given couch document and sql object.
        Return a human-reaedable string describing their differences, or None if the
        two are equivalent.
        """
        raise NotImplementedError()

    @classmethod
    def diff_attr(cls, name, doc, obj, wrap_couch=None, wrap_sql=None):
        """
        Helper for diff_couch_and_sql
        """
        couch = doc.get(name, None)
        sql = getattr(obj, name, None)
        if wrap_couch:
            couch = wrap_couch(couch) if couch is not None else None
        if wrap_sql:
            sql = wrap_sql(sql) if sql is not None else None
        if couch != sql:
            return f"{name}: couch value {couch!r} != sql value {sql!r}"

    @classmethod
    def diff_lists(cls, docs, objects, attr_list):
        diffs = []
        if len(docs) != len(objects):
            diffs.append(f"{len(docs)} in couch != {len(attr_list)} in sql")
        else:
            for couch_field, sql_field in list(zip(docs, objects)):<commit_msg>use correct var in diff message<commit_after>import logging
import sys
import traceback

from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
from django.db import transaction

from corehq.dbaccessors.couchapps.all_docs import get_all_docs_with_doc_types, get_doc_count_by_type
from corehq.util.couchdb_management import couch_config
from corehq.util.django_migrations import skip_on_fresh_install

logger = logging.getLogger(__name__)


class PopulateSQLCommand(BaseCommand):
    """
        Base class for migrating couch docs to sql models.

        Adds a SQL object for any couch doc that doesn't yet have one.
        Override all methods that raise NotImplementedError and, optionoally, couch_db_slug.
    """
    AUTO_MIGRATE_ITEMS_LIMIT = 1000

    @classmethod
    def couch_db_slug(cls):
        # Override this if couch model was not stored in the main commcarehq database
        return None

    @classmethod
    def couch_doc_type(cls):
        raise NotImplementedError()

    @classmethod
    def sql_class(self):
        raise NotImplementedError()

    def update_or_create_sql_object(self, doc):
        """
        This should find and update the sql object that corresponds to the given doc,
        or create it if it doesn't yet exist. This method is responsible for saving
        the sql object.
        """
        raise NotImplementedError()

    @classmethod
    def diff_couch_and_sql(cls, couch, sql):
        """
        This should compare each attribute of the given couch document and sql object.
        Return a human-reaedable string describing their differences, or None if the
        two are equivalent.
        """
        raise NotImplementedError()

    @classmethod
    def diff_attr(cls, name, doc, obj, wrap_couch=None, wrap_sql=None):
        """
        Helper for diff_couch_and_sql
        """
        couch = doc.get(name, None)
        sql = getattr(obj, name, None)
        if wrap_couch:
            couch = wrap_couch(couch) if couch is not None else None
        if wrap_sql:
            sql = wrap_sql(sql) if sql is not None else None
        if couch != sql:
            return f"{name}: couch value {couch!r} != sql value {sql!r}"

    @classmethod
    def diff_lists(cls, docs, objects, attr_list):
        diffs = []
        if len(docs) != len(objects):
            diffs.append(f"{len(docs)} in couch != {len(objects)} in sql")
        else:
            for couch_field, sql_field in list(zip(docs, objects)):
{'fraction_non_alphanumeric': 0.07123070834982193, 'fraction_numerical': 0.0015829046299960427, 'mean_word_length': 2.800751879699248, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '421742', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1429, 'n_words': 486}