<commit_before>import logging
from dimagi.utils.couch.database import iter_docs
from django.utils.encoding import force_unicode
from django.utils.safestring import mark_safe
from corehq.apps.app_manager.const import APP_V1
from corehq.apps.app_manager.detail_screen import get_column_xpath_generator
from corehq.apps.app_manager.models import Application
from corehq.apps.app_manager.xpath import dot_interpolate
import sys
from django.conf import settings

logger = logging.getLogger(__name__)


class AppFilterMigrationMixIn(object):
    """
    This is the main data migration code for the filter changes.
    """

    def get_app_ids(self):
        # this is the only method that migration subclasses should override
        raise NotImplementedError()

    def _get_main_app_ids(self):
        return self._get_app_ids([None, None])

    def _get_released_app_ids(self):
        return self._get_app_ids(['^ReleasedApplications'])

    def _get_all_app_ids(self):
        return self._get_app_ids([None])

    def _get_app_ids(self, startkey):
        return {r['id'] for r in Application.get_db().view(
            'app_manager/applications',
            startkey=startkey,
            endkey=startkey + [{}],
            reduce=False,
        ).all()}

    def forwards(self, orm):
        if not settings.UNIT_TESTING:
            errors = []

            def _migrate_app_ids(app_ids):
                to_save = []
                count = len(app_ids)
                logger.info('migrating {} apps'.format(count))
                for i, app_doc in enumerate(iter_docs(Application.get_db(), app_ids)):
                    try:
                        if app_doc["doc_type"] in ["Application", "Application-Deleted"]:
                            application = Application.wrap(app_doc)
                            should_save = self.migrate_app(application)
                            if should_save:
                                to_save.append(application)
                                if len(to_save) > 25:
                                    self.bulk_save(to_save)
                                    logger.info('completed {}/{} apps'.format(i, count))
                                    to_save = []
                    except Exception:
                        errors.append("App {id} not properly migrated because {error}".format(id=app_doc['_id'],
                                                                                              error=sys.exc_info()[0]))
                if to_save:
                    self.bulk_save(to_save)

            logger.info('migrating applications')
            _migrate_app_ids(self.get_app_ids())

            if errors:
                logger.info('\n'.join(errors))

    @classmethod
    def bulk_save(cls, apps):
        Application.get_db().bulk_save(apps)
        for app in apps:
            logger.info("Filter migration on app {id} complete.".format(id=app.id))

    @classmethod
    def migrate_app(cls, app):
        filter_combination_func = cls.combine_and_interpolate_V2_filters
        if app.application_version == APP_V1:
            filter_combination_func = \
                cls.combine_and_interpolate_V1_filters

        needs_save = False
        for module in app.get_modules():
            detail = module.case_details.short
            # already migrated - don't bother saving again
            if detail.filter:
                return False
            combined_filter_string = filter_combination_func(
                detail.get_columns(), app, module, detail
            )
            detail.filter = combined_filter_string
            if detail.filter:
                needs_save = True
        return needs_save

    @classmethod
    def combine_and_interpolate_V1_filters(cls, columns, app, module, detail):
        """
        Return a single filter xpath generated by ANDing together the given
        componenets. Also replaces "."s with the corresponding xpath. The
        interpolation here is specific to v1 apps! use
        combine_and_interpolate_V2_filters for V2 apps.
        :param column_filters: A list of columns
        :param app:
        :param module:
        :param detail:
        :return:
        """
        filters = []
        for i, column in enumerate(columns):
            if column.format == 'filter':
                value = dot_interpolate(
                    column.filter_xpath,
                    '%s_%s_%s' % (column.model, column.field, i + 1)<commit_msg>Fix migration bug for CarePlan and Advanced Modules<commit_after>import logging
from dimagi.utils.couch.database import iter_docs
from django.utils.encoding import force_unicode
from django.utils.safestring import mark_safe
from corehq.apps.app_manager.const import APP_V1
from corehq.apps.app_manager.detail_screen import get_column_xpath_generator
from corehq.apps.app_manager.models import Application
from corehq.apps.app_manager.xpath import dot_interpolate
import sys
from django.conf import settings

logger = logging.getLogger(__name__)


class AppFilterMigrationMixIn(object):
    """
    This is the main data migration code for the filter changes.
    """

    def get_app_ids(self):
        # this is the only method that migration subclasses should override
        raise NotImplementedError()

    def _get_main_app_ids(self):
        return self._get_app_ids([None, None])

    def _get_released_app_ids(self):
        return self._get_app_ids(['^ReleasedApplications'])

    def _get_all_app_ids(self):
        return self._get_app_ids([None])

    def _get_app_ids(self, startkey):
        return {r['id'] for r in Application.get_db().view(
            'app_manager/applications',
            startkey=startkey,
            endkey=startkey + [{}],
            reduce=False,
        ).all()}

    def forwards(self, orm):
        if not settings.UNIT_TESTING:
            errors = []

            def _migrate_app_ids(app_ids):
                to_save = []
                count = len(app_ids)
                logger.info('migrating {} apps'.format(count))
                for i, app_doc in enumerate(iter_docs(Application.get_db(), app_ids)):
                    try:
                        if app_doc["doc_type"] in ["Application", "Application-Deleted"]:
                            application = Application.wrap(app_doc)
                            should_save = self.migrate_app(application)
                            if should_save:
                                to_save.append(application)
                                if len(to_save) > 25:
                                    self.bulk_save(to_save)
                                    logger.info('completed {}/{} apps'.format(i, count))
                                    to_save = []
                    except Exception:
                        errors.append("App {id} not properly migrated because {error}".format(id=app_doc['_id'],
                                                                                              error=sys.exc_info()[0]))
                if to_save:
                    self.bulk_save(to_save)

            logger.info('migrating applications')
            _migrate_app_ids(self.get_app_ids())

            if errors:
                logger.info('\n'.join(errors))

    @classmethod
    def bulk_save(cls, apps):
        Application.get_db().bulk_save(apps)
        for app in apps:
            logger.info("Filter migration on app {id} complete.".format(id=app.id))

    @classmethod
    def migrate_app(cls, app):
        filter_combination_func = cls.combine_and_interpolate_V2_filters
        if app.application_version == APP_V1:
            filter_combination_func = \
                cls.combine_and_interpolate_V1_filters

        needs_save = False
        for module in app.get_modules():

            for detail_type in ["case_details", "task_details", "goal_details", "product_details"]:
                details = getattr(module, detail_type, None)
                if details is None:
                    # This module does not have the given detail_type
                    continue
                detail = details.short
                # already migrated - don't bother saving again
                if detail.filter:
                    return False
                combined_filter_string = filter_combination_func(
                    detail.get_columns(), app, module, detail
                )
                detail.filter = combined_filter_string
                if detail.filter:
                    needs_save = True
        return needs_save

    @classmethod
    def combine_and_interpolate_V1_filters(cls, columns, app, module, detail):
        """
        Return a single filter xpath generated by ANDing together the given
        componenets. Also replaces "."s with the corresponding xpath. The
        interpolation here is specific to v1 apps! use
        combine_and_interpolate_V2_filters for V2 apps.
        :param column_filters: A list of columns
        :param app:
        :param module:
        :param detail:
        :return:
        """
        filters = []
        for i, column in enumerate(columns):
            if column.format == 'filter':
                value = dot_interpolate(
                    column.filter_xpath,
                    '%s_%s_%s' % (column.model, column.field, i + 1)
{'fraction_non_alphanumeric': 0.08106084910704738, 'fraction_numerical': 0.0025665704202759063, 'mean_word_length': 1.794979079497908, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6951845', 'n_tokens_mistral': 2533, 'n_tokens_neox': 2436, 'n_words': 588}