<commit_before>import copy
from couchdbkit.exceptions import ResourceNotFound
from django.conf import settings
from django.utils.safestring import mark_safe
import re
from corehq.apps.app_manager.models import RemoteApp, Application
from corehq.apps.reports.analytics.couchaccessors import guess_form_name_from_submissions_using_xmlns
from corehq.apps.reports.filters.base import BaseDrilldownOptionFilter, BaseSingleOptionFilter, BaseTagsFilter
from couchforms.analytics import get_all_xmlns_app_id_pairs_submitted_to_in_domain
from dimagi.utils.decorators.memoized import memoized

# For translations
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop, ugettext_lazy

REMOTE_APP_WILDCARD = "http://(.+).commcarehq.org"
MISSING_APP_ID = "_MISSING_APP_ID"

PARAM_SLUG_STATUS = 'status'
PARAM_SLUG_APP_ID = 'app_id'
PARAM_SLUG_MODULE = 'module'
PARAM_SLUG_XMLNS = 'xmlns'


class FormsByApplicationFilter(BaseDrilldownOptionFilter):
    """
        Use this filter to drill down by
        (Active Applications or Deleted Applications or Remote Applications >) Application > Module > Form

        You may also select Unknown Forms for forms that can't be matched to any known Application, Application-Deleted,
        RemoteApp, or RemoteApp-Deleted for this domain.

        You may also hide/show fuzzy results (where fuzzy means you can't match that XMLNS to exactly one Application
        or Application-Deleted).
    """
    slug = "form"
    label = ugettext_lazy("Filter Forms")
    css_class = "span5"
    drilldown_empty_text = ugettext_lazy("You don't have any applications set up, so there are no forms "
                                         "to choose from. Please create an application!")
    template = "reports/filters/form_app_module_drilldown.html"
    unknown_slug = "unknown"
    fuzzy_slug = "@@FUZZY"
    show_global_hide_fuzzy_checkbox = True
    unknown_remote_app_id = 'unknown_remote_app'
    display_app_type = False # whether we're displaying the application type select box in the filter

    @property
    def display_lang(self):
        """
            This should return the lang code of the language being used to view this form.
        """
        return hasattr(self.request, 'couch_user') and self.request.couch_user.language or 'en'

    @property
    def rendered_labels(self):
        """
            Here we determine whether to extend the drilldown to allow the user to choose application types.
            Current supported types are:
            - Active Application (Application)
            - Deleted Application (Application-Deleted)
            - Remote Application (RemoteApp and RemoteApp-Deleted)
        """
        labels = self.get_labels()
        if self.drilldown_map and self.drilldown_map[0]['val'] == 'active':
            labels = [
                 (_('Application Type'),
                  _("Select an Application Type") if self.use_only_last else _("Show all Application Types"),
                  'status'),
                 (_('Application'),
                  _("Select Application...") if self.use_only_last else _("Show all Forms of this Application Type..."),
                  PARAM_SLUG_APP_ID),
             ] + labels[1:]
        return labels

    @property
    def filter_context(self):
        context = super(FormsByApplicationFilter, self).filter_context
        context.update({
            'unknown_available': bool(self._unknown_forms),
            'unknown': {
                'show': self._show_unknown,
                'slug': self.unknown_slug,
                'selected': self._selected_unknown_xmlns,
                'options': self._unknown_forms_options,
                'default_text': "Select an Unknown Form..." if self.use_only_last else "Show All Unknown Forms...",
            },
            'hide_fuzzy': {
                'show': not self._show_unknown and self.show_global_hide_fuzzy_checkbox and self._fuzzy_forms,
                'slug': '%s_%s' % (self.slug, self.fuzzy_slug),
                'checked': self._hide_fuzzy_results,
            },
            'display_app_type': self.display_app_type,
            'support_email': settings.SUPPORT_EMAIL,
        })

        if self.display_app_type and not context['selected']:
            context['selected'] = ['active']
        context["show_advanced"] = self.request.GET.get('show_advanced') == 'on' or context["unknown"]["show"] or \
                                   context["hide_fuzzy"]["checked"] or \
                                   (len(context['selected']) > 0 and context['selected'][0] != 'active')
        return context

    @property
    @memoized
    def drilldown_map(self):
        final_map = []
        map_active = []
        map_remote = []
        map_deleted = []

        all_forms = self._application_forms_info.copy()
        all_forms.update(self._remote_forms_info.copy())

        for app_map in all_forms.values():
            app_langs = app_map['app']['langs']
            is_deleted = app_map['is_deleted']
            is_remote = app_map.get('is_remote', False)

            app_name = self.get_translated_value(self.display_lang, app_langs, app_map['app']['names'])
            if is_deleted:
                app_name = "%s [Deleted Application]" % app_name
            app = self._map_structure(app_map['app']['id'], app_name)

            for module_map in app_map['modules']:
                if module_map['module'] is not None:
                    module_name = self.get_translated_value(
                        self.display_lang, app_langs, module_map['module']['names'])
                    module = self._map_structure(module_map['module']['id'], module_name)
                    for form_map in module_map['forms']:
                        form_name = self.get_translated_value(
                            self.display_lang, app_langs, form_map['form']['names'])
                        module['next'].append(self._map_structure(form_map['xmlns'], form_name))
                    app['next'].append(module)


            if is_remote:
                map_remote.append(app)
            elif is_deleted:
                map_deleted.append(app)
            else:
                map_active.append(app)

        if (bool(map_remote) + bool(map_deleted) + bool(map_active)) > 1:
            self.display_app_type = True
            if map_active:
                final_map.append(self._map_structure('active', _('Active CommCare Applications'), map_active))
            if map_remote:
                final_map.append(self._map_structure('remote', _('Remote CommCare Applications'), map_remote))
            if map_deleted:
                final_map.append(self._map_structure('deleted', _('Deleted CommCare Applications'), map_deleted))
        else:
            final_map.extend(map_active or map_remote or map_deleted)

        return final_map

    @property
    @memoized
    def _all_forms(self):
        """
            Here we grab all forms ever submitted to this domain on CommCare HQ or all forms that the Applications
            for this domain know about.
        """
        form_buckets = get_all_xmlns_app_id_pairs_submitted_to_in_domain(self.domain)
        all_submitted = {self.make_xmlns_app_key(xmlns, app_id)
                         for xmlns, app_id in form_buckets}
        from_apps = set(self._application_forms)
        return list(all_submitted.union(from_apps))

    @property<commit_msg>add constants for values<commit_after>import copy
from couchdbkit.exceptions import ResourceNotFound
from django.conf import settings
from django.utils.safestring import mark_safe
import re
from corehq.apps.app_manager.models import RemoteApp, Application
from corehq.apps.reports.analytics.couchaccessors import guess_form_name_from_submissions_using_xmlns
from corehq.apps.reports.filters.base import BaseDrilldownOptionFilter, BaseSingleOptionFilter, BaseTagsFilter
from couchforms.analytics import get_all_xmlns_app_id_pairs_submitted_to_in_domain
from dimagi.utils.decorators.memoized import memoized

# For translations
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop, ugettext_lazy

REMOTE_APP_WILDCARD = "http://(.+).commcarehq.org"
MISSING_APP_ID = "_MISSING_APP_ID"

PARAM_SLUG_STATUS = 'status'
PARAM_SLUG_APP_ID = 'app_id'
PARAM_SLUG_MODULE = 'module'
PARAM_SLUG_XMLNS = 'xmlns'

PARAM_VALUE_STATUS_ACTIVE = 'active'
PARAM_VALUE_STATUS_REMOTE = 'remote'
PARAM_VALUE_STATUS_DELETED = 'deleted'


class FormsByApplicationFilter(BaseDrilldownOptionFilter):
    """
        Use this filter to drill down by
        (Active Applications or Deleted Applications or Remote Applications >) Application > Module > Form

        You may also select Unknown Forms for forms that can't be matched to any known Application, Application-Deleted,
        RemoteApp, or RemoteApp-Deleted for this domain.

        You may also hide/show fuzzy results (where fuzzy means you can't match that XMLNS to exactly one Application
        or Application-Deleted).
    """
    slug = "form"
    label = ugettext_lazy("Filter Forms")
    css_class = "span5"
    drilldown_empty_text = ugettext_lazy("You don't have any applications set up, so there are no forms "
                                         "to choose from. Please create an application!")
    template = "reports/filters/form_app_module_drilldown.html"
    unknown_slug = "unknown"
    fuzzy_slug = "@@FUZZY"
    show_global_hide_fuzzy_checkbox = True
    unknown_remote_app_id = 'unknown_remote_app'
    display_app_type = False # whether we're displaying the application type select box in the filter

    @property
    def display_lang(self):
        """
            This should return the lang code of the language being used to view this form.
        """
        return hasattr(self.request, 'couch_user') and self.request.couch_user.language or 'en'

    @property
    def rendered_labels(self):
        """
            Here we determine whether to extend the drilldown to allow the user to choose application types.
            Current supported types are:
            - Active Application (Application)
            - Deleted Application (Application-Deleted)
            - Remote Application (RemoteApp and RemoteApp-Deleted)
        """
        labels = self.get_labels()
        if self.drilldown_map and self.drilldown_map[0]['val'] == PARAM_VALUE_STATUS_ACTIVE:
            labels = [
                 (_('Application Type'),
                  _("Select an Application Type") if self.use_only_last else _("Show all Application Types"),
                  'status'),
                 (_('Application'),
                  _("Select Application...") if self.use_only_last else _("Show all Forms of this Application Type..."),
                  PARAM_SLUG_APP_ID),
             ] + labels[1:]
        return labels

    @property
    def filter_context(self):
        context = super(FormsByApplicationFilter, self).filter_context
        context.update({
            'unknown_available': bool(self._unknown_forms),
            'unknown': {
                'show': self._show_unknown,
                'slug': self.unknown_slug,
                'selected': self._selected_unknown_xmlns,
                'options': self._unknown_forms_options,
                'default_text': "Select an Unknown Form..." if self.use_only_last else "Show All Unknown Forms...",
            },
            'hide_fuzzy': {
                'show': not self._show_unknown and self.show_global_hide_fuzzy_checkbox and self._fuzzy_forms,
                'slug': '%s_%s' % (self.slug, self.fuzzy_slug),
                'checked': self._hide_fuzzy_results,
            },
            'display_app_type': self.display_app_type,
            'support_email': settings.SUPPORT_EMAIL,
        })

        if self.display_app_type and not context['selected']:
            context['selected'] = [PARAM_VALUE_STATUS_ACTIVE]
        context["show_advanced"] = self.request.GET.get('show_advanced') == 'on' or context["unknown"]["show"] or \
                                   context["hide_fuzzy"]["checked"] or \
                                   (len(context['selected']) > 0 and context['selected'][0] != PARAM_VALUE_STATUS_ACTIVE)
        return context

    @property
    @memoized
    def drilldown_map(self):
        final_map = []
        map_active = []
        map_remote = []
        map_deleted = []

        all_forms = self._application_forms_info.copy()
        all_forms.update(self._remote_forms_info.copy())

        for app_map in all_forms.values():
            app_langs = app_map['app']['langs']
            is_deleted = app_map['is_deleted']
            is_remote = app_map.get('is_remote', False)

            app_name = self.get_translated_value(self.display_lang, app_langs, app_map['app']['names'])
            if is_deleted:
                app_name = "%s [Deleted Application]" % app_name
            app = self._map_structure(app_map['app']['id'], app_name)

            for module_map in app_map['modules']:
                if module_map['module'] is not None:
                    module_name = self.get_translated_value(
                        self.display_lang, app_langs, module_map['module']['names'])
                    module = self._map_structure(module_map['module']['id'], module_name)
                    for form_map in module_map['forms']:
                        form_name = self.get_translated_value(
                            self.display_lang, app_langs, form_map['form']['names'])
                        module['next'].append(self._map_structure(form_map['xmlns'], form_name))
                    app['next'].append(module)


            if is_remote:
                map_remote.append(app)
            elif is_deleted:
                map_deleted.append(app)
            else:
                map_active.append(app)

        if (bool(map_remote) + bool(map_deleted) + bool(map_active)) > 1:
            self.display_app_type = True
            if map_active:
                final_map.append(self._map_structure(PARAM_VALUE_STATUS_ACTIVE, _('Active CommCare Applications'), map_active))
            if map_remote:
                final_map.append(self._map_structure(PARAM_VALUE_STATUS_REMOTE, _('Remote CommCare Applications'), map_remote))
            if map_deleted:
                final_map.append(self._map_structure(PARAM_VALUE_STATUS_DELETED, _('Deleted CommCare Applications'), map_deleted))
        else:
            final_map.extend(map_active or map_remote or map_deleted)

        return final_map

    @property
    @memoized
    def _all_forms(self):
        """
            Here we grab all forms ever submitted to this domain on CommCare HQ or all forms that the Applications
            for this domain know about.
        """
        form_buckets = get_all_xmlns_app_id_pairs_submitted_to_in_domain(self.domain)
        all_submitted = {self.make_xmlns_app_key(xmlns, app_id)
                         for xmlns, app_id in form_buckets}
        from_apps = set(self._application_forms)
        return list(all_submitted.union(from_apps))

    @property
{'fraction_non_alphanumeric': 0.09099294985833828, 'fraction_numerical': 0.000790670092903736, 'mean_word_length': 2.466879853814527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6942779', 'n_tokens_mistral': 4333, 'n_tokens_neox': 4179, 'n_words': 1000}