<commit_before>import re

from collections import defaultdict

from django.utils.functional import cached_property
from django.utils.translation import ugettext as _

from casexml.apps.case.models import CommCareCase
from dimagi.utils.logging import notify_exception

from corehq.apps.app_manager.dbaccessors import get_app_cached
from corehq.apps.app_manager.util import module_offers_search
from corehq.apps.case_search.const import CASE_SEARCH_MAX_RESULTS
from corehq.apps.case_search.exceptions import CaseSearchUserError
from corehq.apps.case_search.filter_dsl import (
    CaseFilterError,
    TooManyRelatedCasesError,
)
from corehq.apps.case_search.models import (
    CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY,
    CASE_SEARCH_XPATH_QUERY_KEY,
    SEARCH_QUERY_CUSTOM_VALUE,
    UNSEARCHABLE_KEYS,
    CaseSearchConfig,
)
from corehq.apps.es import queries
from corehq.apps.es.case_search import (
    CaseSearchES,
    case_property_range_query,
    flatten_result,
)


def get_case_search_results(domain, criteria, app_id=None):
    try:
        # could be a list or a single string
        case_type = criteria.pop('case_type')
        case_types = case_type if isinstance(case_type, list) else [case_type]
    except KeyError:
        raise CaseSearchUserError(_('Search request must specify case type'))

    try:
        case_search_criteria = CaseSearchCriteria(domain, case_types, criteria)
    except TooManyRelatedCasesError:
        raise CaseSearchUserError(_('Search has too many results. Please try a more specific search.'))
    except CaseFilterError as e:
        # This is an app building error, notify so we can track
        notify_exception(None, str(e), details=dict(
            exception_type=type(e),
        ))
        raise CaseSearchUserError(str(e))
    search_es = case_search_criteria.search_es

    try:
        hits = search_es.run().raw_hits
    except Exception as e:
        notify_exception(None, str(e), details=dict(
            exception_type=type(e),
        ))
        raise

    # Even if it's a SQL domain, we just need to render the hits as cases, so CommCareCase.wrap will be fine
    cases = [CommCareCase.wrap(flatten_result(result, include_score=True)) for result in hits]
    if app_id:
        cases.extend(get_related_cases(domain, app_id, case_types, cases))
    return cases


class CaseSearchCriteria(object):
    """Compiles the case search object for the view
    """

    def __init__(self, domain, case_types, criteria):
        self.domain = domain
        self.case_types = case_types
        self.criteria = criteria

        self.config = self._get_config()
        self.search_es = self._get_initial_search_es()

        self._assemble_optional_search_params()

    def _get_config(self):
        try:
            config = (CaseSearchConfig.objects
                      .prefetch_related('fuzzy_properties')
                      .prefetch_related('ignore_patterns')
                      .get(domain=self.domain))
        except CaseSearchConfig.DoesNotExist as e:
            from corehq.util.soft_assert import soft_assert
            _soft_assert = soft_assert(
                to="{}<EMAIL>".format('frener', 'dimagi'),
                notify_admins=False, send_to_ops=False
            )
            _soft_assert(
                False,
                "Someone in domain: {} tried accessing case search without a config".format(self.domain),
                e
            )
            config = CaseSearchConfig(domain=self.domain)
        return config

    def _get_initial_search_es(self):
        search_es = (CaseSearchES()
                     .domain(self.domain)
                     .case_type(self.case_types)
                     .is_closed(False)
                     .size(CASE_SEARCH_MAX_RESULTS)
                     .set_sorting_block(['_score', '_doc']))
        return search_es

    def _assemble_optional_search_params(self):
        self._validate_multiple_parameter_values()
        self._add_xpath_query()
        self._add_owner_id()
        self._add_blacklisted_owner_ids()
        self._add_case_property_queries()

    def _validate_multiple_parameter_values(self):
        disallowed_multiple_value_parameters = [
            CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY,
            'owner_id',
            CASE_SEARCH_XPATH_QUERY_KEY,
        ]

        for key, val in self.criteria.items():
            if not isinstance(val, list):
                continue
            is_daterange = any([v.startswith('__range__') for v in val])
            if key in disallowed_multiple_value_parameters or '/' in key or is_daterange:
                raise CaseFilterError(
                    _("Multiple values are only supported for simple text and range searches"),
                    key
                )

    def _add_xpath_query(self):
        query = self.criteria.pop(CASE_SEARCH_XPATH_QUERY_KEY, None)
        if query:
            self.search_es = self.search_es.xpath_query(self.domain, query)

    def _add_owner_id(self):
        owner_id = self.criteria.pop('owner_id', False)
        if owner_id:
            self.search_es = self.search_es.owner(owner_id)

    def _add_blacklisted_owner_ids(self):
        blacklisted_owner_ids = self.criteria.pop(CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY, None)
        if blacklisted_owner_ids is not None:
            for blacklisted_owner_id in blacklisted_owner_ids.split(' '):
                self.search_es = self.search_es.blacklist_owner_id(blacklisted_owner_id)

    def _add_daterange_query(self, key, value):
        # Add query for specially formatted daterange param
        #   The format is __range__YYYY-MM-DD__YYYY-MM-DD, which is
        #   used by App manager case-search feature
        if not isinstance(value, list):
            pattern = re.compile(r'__range__\d{4}-\d{2}-\d{2}__\d{4}-\d{2}-\d{2}')
            match = pattern.match(value)
            if match:
                _, _, startdate, enddate = value.split('__')
                self.search_es = self.search_es.add_query(
                    case_property_range_query(key, gte=startdate, lte=enddate),
                    queries.MUST,
                )

    def _add_case_property_queries(self):
        for key, value in self.criteria.items():
            if key in UNSEARCHABLE_KEYS or key.startswith(SEARCH_QUERY_CUSTOM_VALUE):
                continue
            if not isinstance(value, list) and value.startswith('__range__'):
                self._add_daterange_query(key, value)
                continue

            value = self._remove_ignored_patterns(key, value)
            fuzzy = key in self._fuzzy_properties
            if '/' in key:
                query = '{} = "{}"'.format(key, value)
                self.search_es = self.search_es.xpath_query(self.domain, query, fuzzy=fuzzy)
            else:
                self.search_es = self.search_es.case_property_query(key, value, fuzzy=fuzzy)

    def _remove_ignored_patterns(self, case_property, value):
        for to_remove in self._patterns_to_remove[case_property]:
            if isinstance(value, list):
                value = [re.sub(to_remove, '', val) for val in value]
            else:
                value = re.sub(to_remove, '', value)
        return value

    @cached_property
    def _patterns_to_remove(self):
        patterns_by_property = defaultdict(list)
        for pattern in self.config.ignore_patterns.filter(domain=self.domain, case_type__in=self.case_types):
            patterns_by_property[pattern.case_property].append(re.escape(pattern.regex))
        return patterns_by_property

    @cached_property
    def _fuzzy_properties(self):
        return [
            prop for properties_config in
            self.config.fuzzy_properties.filter(domain=self.domain, case_type__in=self.case_types)
            for prop in properties_config.properties
        ]
<commit_msg>Pull out query construction from mutation<commit_after>import re
from collections import defaultdict

from django.utils.functional import cached_property
from django.utils.translation import ugettext as _

from casexml.apps.case.models import CommCareCase
from dimagi.utils.logging import notify_exception

from corehq.apps.app_manager.dbaccessors import get_app_cached
from corehq.apps.app_manager.util import module_offers_search
from corehq.apps.case_search.const import CASE_SEARCH_MAX_RESULTS
from corehq.apps.case_search.exceptions import CaseSearchUserError
from corehq.apps.case_search.filter_dsl import (
    CaseFilterError,
    TooManyRelatedCasesError,
    build_filter_from_xpath,
)
from corehq.apps.case_search.models import (
    CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY,
    CASE_SEARCH_XPATH_QUERY_KEY,
    SEARCH_QUERY_CUSTOM_VALUE,
    UNSEARCHABLE_KEYS,
    CaseSearchConfig,
)
from corehq.apps.es import queries
from corehq.apps.es.case_search import (
    CaseSearchES,
    case_property_query,
    case_property_range_query,
    flatten_result,
)


def get_case_search_results(domain, criteria, app_id=None):
    try:
        # could be a list or a single string
        case_type = criteria.pop('case_type')
        case_types = case_type if isinstance(case_type, list) else [case_type]
    except KeyError:
        raise CaseSearchUserError(_('Search request must specify case type'))

    try:
        case_search_criteria = CaseSearchCriteria(domain, case_types, criteria)
    except TooManyRelatedCasesError:
        raise CaseSearchUserError(_('Search has too many results. Please try a more specific search.'))
    except CaseFilterError as e:
        # This is an app building error, notify so we can track
        notify_exception(None, str(e), details=dict(
            exception_type=type(e),
        ))
        raise CaseSearchUserError(str(e))
    search_es = case_search_criteria.search_es

    try:
        hits = search_es.run().raw_hits
    except Exception as e:
        notify_exception(None, str(e), details=dict(
            exception_type=type(e),
        ))
        raise

    # Even if it's a SQL domain, we just need to render the hits as cases, so CommCareCase.wrap will be fine
    cases = [CommCareCase.wrap(flatten_result(result, include_score=True)) for result in hits]
    if app_id:
        cases.extend(get_related_cases(domain, app_id, case_types, cases))
    return cases


class CaseSearchCriteria(object):
    """Compiles the case search object for the view
    """

    def __init__(self, domain, case_types, criteria):
        self.domain = domain
        self.case_types = case_types
        self.criteria = criteria

        self.config = self._get_config()
        self.search_es = self._get_initial_search_es()

        self._assemble_optional_search_params()

    def _get_config(self):
        try:
            config = (CaseSearchConfig.objects
                      .prefetch_related('fuzzy_properties')
                      .prefetch_related('ignore_patterns')
                      .get(domain=self.domain))
        except CaseSearchConfig.DoesNotExist as e:
            from corehq.util.soft_assert import soft_assert
            _soft_assert = soft_assert(
                to="{}<EMAIL>".format('frener', 'dimagi'),
                notify_admins=False, send_to_ops=False
            )
            _soft_assert(
                False,
                "Someone in domain: {} tried accessing case search without a config".format(self.domain),
                e
            )
            config = CaseSearchConfig(domain=self.domain)
        return config

    def _get_initial_search_es(self):
        search_es = (CaseSearchES()
                     .domain(self.domain)
                     .case_type(self.case_types)
                     .is_closed(False)
                     .size(CASE_SEARCH_MAX_RESULTS)
                     .set_sorting_block(['_score', '_doc']))
        return search_es

    def _assemble_optional_search_params(self):
        self._validate_multiple_parameter_values()
        self._add_xpath_query()
        self._add_owner_id()
        self._add_blacklisted_owner_ids()
        self._add_case_property_queries()

    def _validate_multiple_parameter_values(self):
        disallowed_multiple_value_parameters = [
            CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY,
            'owner_id',
            CASE_SEARCH_XPATH_QUERY_KEY,
        ]

        for key, val in self.criteria.items():
            if not isinstance(val, list):
                continue
            is_daterange = any([v.startswith('__range__') for v in val])
            if key in disallowed_multiple_value_parameters or '/' in key or is_daterange:
                raise CaseFilterError(
                    _("Multiple values are only supported for simple text and range searches"),
                    key
                )

    def _add_xpath_query(self):
        query = self.criteria.pop(CASE_SEARCH_XPATH_QUERY_KEY, None)
        if query:
            self.search_es = self.search_es.xpath_query(self.domain, query)

    def _add_owner_id(self):
        owner_id = self.criteria.pop('owner_id', False)
        if owner_id:
            self.search_es = self.search_es.owner(owner_id)

    def _add_blacklisted_owner_ids(self):
        blacklisted_owner_ids = self.criteria.pop(CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY, None)
        if blacklisted_owner_ids is not None:
            for blacklisted_owner_id in blacklisted_owner_ids.split(' '):
                self.search_es = self.search_es.blacklist_owner_id(blacklisted_owner_id)

    def _get_daterange_query(self, key, value):
        # Add query for specially formatted daterange param
        #   The format is __range__YYYY-MM-DD__YYYY-MM-DD, which is
        #   used by App manager case-search feature
        if not isinstance(value, list):
            pattern = re.compile(r'__range__\d{4}-\d{2}-\d{2}__\d{4}-\d{2}-\d{2}')
            match = pattern.match(value)
            if match:
                _, _, startdate, enddate = value.split('__')
                return case_property_range_query(key, gte=startdate, lte=enddate),

    def _add_case_property_queries(self):
        for key, value in self.criteria.items():
            if key in UNSEARCHABLE_KEYS or key.startswith(SEARCH_QUERY_CUSTOM_VALUE):
                continue
            self.search_es = self.search_es.add_query(self._get_query(key, value), queries.MUST)

    def _get_query(self, key, value):
        if not isinstance(value, list) and value.startswith('__range__'):
            return self._get_daterange_query(key, value)

        value = self._remove_ignored_patterns(key, value)
        fuzzy = key in self._fuzzy_properties
        if '/' in key:
            query = '{} = "{}"'.format(key, value)
            return build_filter_from_xpath(self.domain, query, fuzzy=fuzzy),
        else:
            return case_property_query(key, value, fuzzy=fuzzy)

    def _remove_ignored_patterns(self, case_property, value):
        for to_remove in self._patterns_to_remove[case_property]:
            if isinstance(value, list):
                value = [re.sub(to_remove, '', val) for val in value]
            else:
                value = re.sub(to_remove, '', value)
        return value

    @cached_property
    def _patterns_to_remove(self):
        patterns_by_property = defaultdict(list)
        for pattern in self.config.ignore_patterns.filter(domain=self.domain, case_type__in=self.case_types):
            patterns_by_property[pattern.case_property].append(re.escape(pattern.regex))
        return patterns_by_property

    @cached_property
    def _fuzzy_properties(self):
        return [
            prop for properties_config in
            self.config.fuzzy_properties.filter(domain=self.domain, case_type__in=self.case_types)
            for prop in properties_config.properties
        ]

{'fraction_non_alphanumeric': 0.08034923446792357, 'fraction_numerical': 0.0007592053650512464, 'mean_word_length': 2.536241610738255, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3200263', 'n_tokens_mistral': 4749, 'n_tokens_neox': 4592, 'n_words': 981}