<commit_before>import logging
import simplejson

from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop

from couchdbkit import RequestFailed
from dimagi.utils.decorators.memoized import memoized

from corehq.apps.api.es import CaseES
from corehq.apps.groups.models import Group
from corehq.apps.reports.api import ReportDataSource
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.filters.search import SearchFilter
from corehq.apps.reports.filters.select import SelectOpenCloseFilter
from corehq.apps.reports.filters.users import (ExpandedMobileWorkerFilter,
        SelectMobileWorkerFilter, ExpandedMobileWorkerFilterWithAllData)
from corehq.apps.reports.generic import ElasticProjectInspectionReport
from corehq.apps.reports.models import HQUserType
from corehq.apps.reports.standard import ProjectReportParametersMixin
from corehq.apps.reports.standard.inspect import ProjectInspectionReport

from .data_sources import CaseInfo, CaseDisplay

class CaseListMixin(ElasticProjectInspectionReport, ProjectReportParametersMixin):
    fields = [
        'corehq.apps.reports.filters.users.ExpandedMobileWorkerFilterWithAllData',
        'corehq.apps.reports.filters.select.CaseTypeFilter',
        'corehq.apps.reports.filters.select.SelectOpenCloseFilter',
        'corehq.apps.reports.standard.cases.filters.CaseSearchFilter',
    ]

    case_filter = {}
    ajax_pagination = True
    asynchronous = True

    @property
    @memoized
    def case_es(self):
        return CaseES(self.domain)

    def build_query(self, case_type=None, afilter=None, status=None, owner_ids=None, user_ids=None, search_string=None):
        owner_ids = owner_ids or []
        user_ids = user_ids or []

        def _filter_gen(key, flist):
            return {"terms": {
                key: [item.lower() for item in flist if item]
            }}

        def _domain_term():
            return {"term": {"domain.exact": self.domain}}

        subterms = [_domain_term(), afilter] if afilter else [_domain_term()]
        if case_type:
            subterms.append({"term": {"type.exact": case_type}})

        if status:
            subterms.append({"term": {"closed": (status == 'closed')}})


        if not ExpandedMobileWorkerFilterWithAllData.show_all_data(self.request):
            owner_filters = _filter_gen('owner_id', owner_ids)
            user_filters = _filter_gen('user_id', user_ids)
            filters = filter(None, [owner_filters, user_filters])
            if filters:
                subterms.append({'or': filters})

        if search_string:
            query_block = {
                "query_string": {"query": search_string}}  # todo, make sure this doesn't suck
        else:
            query_block = {"match_all": {}}

        and_block = {'and': subterms} if subterms else {}

        es_query = {
            'query': {
                'filtered': {
                    'query': query_block,
                    'filter': and_block
                }
            },
            'sort': self.get_sorting_block(),
            'from': self.pagination.start,
            'size': self.pagination.count,
        }
        return es_query

    @property
    @memoized
    def es_results(self):
        case_es = self.case_es
        user_ids, owner_ids = self.case_users_and_owners
        query = self.build_query(case_type=self.case_type, afilter=self.case_filter,
                                 status=self.case_status, owner_ids=owner_ids+user_ids, user_ids=user_ids,
                                 search_string=SearchFilter.get_value(self.request, self.domain))
        query_results = case_es.run_query(query)

        if query_results is None or 'hits' not in query_results:
            logging.error("CaseListMixin query error: %s, urlpath: %s, params: %s, user: %s yielded a result indicating a query error: %s, results: %s" % (
                self.__class__.__name__,
                self.request.path,
                self.request.GET.urlencode(),
                self.request.couch_user.username,
                simplejson.dumps(query),
                simplejson.dumps(query_results)
            ))
            raise RequestFailed
        return query_results

    @property
    @memoized
    def case_users_and_owners(self):
        users_data = ExpandedMobileWorkerFilter.pull_users_from_es(
            self.domain, self.request, fields=[])
        user_ids = filter(None, [u["_id"] for u in users_data["hits"]["hits"]])
        group_owner_ids = []
        for user_id in user_ids:
            group_owner_ids.extend([
                group._id
                for group in Group.by_user(user_id)
                if group.case_sharing
            ])
        if HQUserType.COMMTRACK in ExpandedMobileWorkerFilter.user_types(self.request):
            user_ids.append("commtrack-system")
        return user_ids, filter(None, group_owner_ids)

    def get_case(self, row):
        if '_source' in row:
            case_dict = row['_source']<commit_msg>Replaced remaning references to ExpandedMobileWorkerFilter<commit_after>import logging
import simplejson

from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop

from couchdbkit import RequestFailed
from dimagi.utils.decorators.memoized import memoized

from corehq.apps.api.es import CaseES
from corehq.apps.groups.models import Group
from corehq.apps.reports.api import ReportDataSource
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.filters.search import SearchFilter
from corehq.apps.reports.filters.select import SelectOpenCloseFilter
from corehq.apps.reports.filters.users import SelectMobileWorkerFilter,\
    ExpandedMobileWorkerFilterWithAllData
from corehq.apps.reports.generic import ElasticProjectInspectionReport
from corehq.apps.reports.models import HQUserType
from corehq.apps.reports.standard import ProjectReportParametersMixin
from corehq.apps.reports.standard.inspect import ProjectInspectionReport

from .data_sources import CaseInfo, CaseDisplay


class CaseListMixin(ElasticProjectInspectionReport, ProjectReportParametersMixin):
    fields = [
        'corehq.apps.reports.filters.users.ExpandedMobileWorkerFilterWithAllData',
        'corehq.apps.reports.filters.select.CaseTypeFilter',
        'corehq.apps.reports.filters.select.SelectOpenCloseFilter',
        'corehq.apps.reports.standard.cases.filters.CaseSearchFilter',
    ]

    case_filter = {}
    ajax_pagination = True
    asynchronous = True

    @property
    @memoized
    def case_es(self):
        return CaseES(self.domain)

    def build_query(self, case_type=None, afilter=None, status=None, owner_ids=None, user_ids=None, search_string=None):
        owner_ids = owner_ids or []
        user_ids = user_ids or []

        def _filter_gen(key, flist):
            return {"terms": {
                key: [item.lower() for item in flist if item]
            }}

        def _domain_term():
            return {"term": {"domain.exact": self.domain}}

        subterms = [_domain_term(), afilter] if afilter else [_domain_term()]
        if case_type:
            subterms.append({"term": {"type.exact": case_type}})

        if status:
            subterms.append({"term": {"closed": (status == 'closed')}})


        if not ExpandedMobileWorkerFilterWithAllData.show_all_data(self.request):
            owner_filters = _filter_gen('owner_id', owner_ids)
            user_filters = _filter_gen('user_id', user_ids)
            filters = filter(None, [owner_filters, user_filters])
            if filters:
                subterms.append({'or': filters})

        if search_string:
            query_block = {
                "query_string": {"query": search_string}}  # todo, make sure this doesn't suck
        else:
            query_block = {"match_all": {}}

        and_block = {'and': subterms} if subterms else {}

        es_query = {
            'query': {
                'filtered': {
                    'query': query_block,
                    'filter': and_block
                }
            },
            'sort': self.get_sorting_block(),
            'from': self.pagination.start,
            'size': self.pagination.count,
        }
        return es_query

    @property
    @memoized
    def es_results(self):
        case_es = self.case_es
        user_ids, owner_ids = self.case_users_and_owners
        query = self.build_query(case_type=self.case_type, afilter=self.case_filter,
                                 status=self.case_status, owner_ids=owner_ids+user_ids, user_ids=user_ids,
                                 search_string=SearchFilter.get_value(self.request, self.domain))
        query_results = case_es.run_query(query)

        if query_results is None or 'hits' not in query_results:
            logging.error("CaseListMixin query error: %s, urlpath: %s, params: %s, user: %s yielded a result indicating a query error: %s, results: %s" % (
                self.__class__.__name__,
                self.request.path,
                self.request.GET.urlencode(),
                self.request.couch_user.username,
                simplejson.dumps(query),
                simplejson.dumps(query_results)
            ))
            raise RequestFailed
        return query_results

    @property
    @memoized
    def case_users_and_owners(self):
        users_data = ExpandedMobileWorkerFilterWithAllData.pull_users_from_es(
            self.domain, self.request, fields=[])
        user_ids = filter(None, [u["_id"] for u in users_data["hits"]["hits"]])
        group_owner_ids = []
        for user_id in user_ids:
            group_owner_ids.extend([
                group._id
                for group in Group.by_user(user_id)
                if group.case_sharing
            ])
        if HQUserType.COMMTRACK in ExpandedMobileWorkerFilterWithAllData.user_types(self.request):
            user_ids.append("commtrack-system")
        return user_ids, filter(None, group_owner_ids)

    def get_case(self, row):
        if '_source' in row:
            case_dict = row['_source']
{'fraction_non_alphanumeric': 0.0929320654837761, 'fraction_numerical': 0.0, 'mean_word_length': 2.5646401118099234, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2390846', 'n_tokens_mistral': 2999, 'n_tokens_neox': 2874, 'n_words': 604}