<commit_before>from _collections import defaultdict
import datetime
import json
import logging
import sys
import dateutil
from dimagi.utils.decorators.memoized import memoized
from django.conf import settings
from django.core.urlresolvers import reverse, NoReverseMatch
from django.http import HttpResponseBadRequest, Http404
from django.template.defaultfilters import yesno
from django.utils import html
import pytz
from restkit.errors import RequestFailed
from casexml.apps.case.models import CommCareCase
from corehq.apps.app_manager.models import Application
from corehq.apps.hqsofabed.models import HQFormData
from corehq.apps.reports import util
from corehq.apps.reports.calc import entrytimes
from corehq.apps.reports.custom import HQReport
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn, DTSortType
from corehq.apps.reports.display import xmlns_to_name, FormType
from corehq.apps.reports.fields import FilterUsersField, CaseTypeField, SelectMobileWorkerField
from corehq.apps.reports.models import HQUserType, FormExportSchema
from couchexport.models import SavedExportSchema
from couchforms.models import XFormInstance
from dimagi.utils.couch.database import get_db
from dimagi.utils.couch.pagination import DatatablesParams
from dimagi.utils.dates import DateSpan, force_to_datetime
from dimagi.utils.parsing import json_format_datetime, string_to_datetime
from dimagi.utils.timezones import utils as tz_utils
from dimagi.utils.web import json_request, get_url_base

DATE_FORMAT = "%Y-%m-%d"

class StandardHQReport(HQReport):
    user_filter = HQUserType.use_defaults()
    fields = ['corehq.apps.reports.fields.FilterUsersField']
    history = None
    use_json = False
    hide_filters = False
    custom_breadcrumbs = None

    def get_global_params(self):
        self.request_params = json_request(self.request.GET)
        # the history param lets you run a report as if it were a given time in the past
        hist_param = self.request_params.get('history', None)
        if hist_param:
            self.history = datetime.datetime.strptime(hist_param, DATE_FORMAT)
        self.user_filter, _ = FilterUsersField.get_user_filter(self.request)

        self.group = self.request_params.get('group','')
        self.individual = self.request_params.get('individual', '')
        self.case_type = self.request_params.get('case_type', '')

        self.users = util.get_all_users_by_domain(self.domain, self.group, self.individual, self.user_filter)

        if self.individual:
            self.name = "%s for %s" % (self.name, self.users[0].raw_username)
        self.get_parameters()
        self.context['report_hide_filters'] = self.hide_filters
        self.context['report_breadcrumbs'] = self.custom_breadcrumbs

    def get_parameters(self):
        # here's where you can define any extra parameters you want to process before
        # proceeding with the rest of get_report_context
        pass

    def get_report_context(self):
        self.context['standard_report'] = True
        self.context['layout_flush_content'] = True
        self.build_selector_form()
        self.context.update(util.report_context(self.domain,
                                        report_partial = self.report_partial,
                                        title = self.name,
                                        headers = self.headers,
                                        rows = self.rows,
                                        show_time_notice = self.show_time_notice
                                      ))
    
    def as_view(self):
        self.get_global_params()
        return super(StandardHQReport, self).as_view()

    def as_json(self):
        self.get_global_params()
        return super(StandardHQReport, self).as_json()


class StandardTabularHQReport(StandardHQReport):
    total_row = None
    default_rows = 10
    start_at_row = 0

#    exportable = True
    def get_headers(self):
        return DataTablesHeader()

    def get_rows(self):
        return self.rows

    def get_report_context(self):
        self.context['report_datatables'] = {
            "defaultNumRows": self.default_rows,
            "startAtRowNum": self.start_at_row
        }
        if self.use_json:
            self.context['ajax_source'] = reverse('json_report_dispatcher',
                                                  args=[self.domain, self.slug])
            self.context['ajax_params'] = [{'name': 'individual', 'value': self.individual},
                                            {'name': 'group', 'value': self.group},
                                            {'name': 'casetype', 'value': self.case_type},
                                            {'name': 'ufilter', 'value': [f.type for f in self.user_filter if f.show]}]
        else:
            self.context['ajax_source'] = False
            self.context['ajax_params'] = False

        self.headers = self.get_headers()
        self.rows = self.get_rows()

        self.context['header'] = self.headers
        self.context['rows'] = self.rows
        if self.total_row:
            self.context['total_row'] = self.total_row

        super(StandardTabularHQReport, self).get_report_context()


class PaginatedHistoryHQReport(StandardTabularHQReport):
    use_json = True
    fields = ['corehq.apps.reports.fields.FilterUsersField',
              'corehq.apps.reports.fields.SelectMobileWorkerField']

    def get_parameters(self):
        self.userIDs = [user.user_id for user in self.users if user.user_id]
        self.usernames = dict([(user.user_id, user.username_in_report) for user in self.users])

    def json_data(self):
        params = DatatablesParams.from_request_dict(self.request.REQUEST)
        rows = self.paginate_rows(params.start, params.count)
        return {
            "sEcho": params.echo,
            "iTotalDisplayRecords": self.count,
            "iTotalRecords": self.count,
            "aaData": rows
        }

    def paginate_rows(self, skip, limit):
        # gather paginated rows here
        self.count = 0
        return []


class StandardDateHQReport(StandardHQReport):

    def __init__(self, domain, request, base_context=None):
        base_context = base_context or {}
        super(StandardDateHQReport, self).__init__(domain, request, base_context)
        self.datespan = self.get_default_datespan()
        self.datespan.is_default = True

    def get_default_datespan(self):
        return DateSpan.since(7, format="%Y-%m-%d", timezone=self.timezone)

    def get_global_params(self):
        if self.request.datespan.is_valid() and not self.request.datespan.is_default:
            self.datespan.enddate = self.request.datespan.enddate<commit_msg>add total_count property for better ui cues about filtered table data<commit_after>from _collections import defaultdict
import datetime
import json
import logging
import sys
import dateutil
from dimagi.utils.decorators.memoized import memoized
from django.conf import settings
from django.core.urlresolvers import reverse, NoReverseMatch
from django.http import HttpResponseBadRequest, Http404
from django.template.defaultfilters import yesno
from django.utils import html
import pytz
from restkit.errors import RequestFailed
from casexml.apps.case.models import CommCareCase
from corehq.apps.app_manager.models import Application
from corehq.apps.hqsofabed.models import HQFormData
from corehq.apps.reports import util
from corehq.apps.reports.calc import entrytimes
from corehq.apps.reports.custom import HQReport
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn, DTSortType
from corehq.apps.reports.display import xmlns_to_name, FormType
from corehq.apps.reports.fields import FilterUsersField, CaseTypeField, SelectMobileWorkerField
from corehq.apps.reports.models import HQUserType, FormExportSchema
from couchexport.models import SavedExportSchema
from couchforms.models import XFormInstance
from dimagi.utils.couch.database import get_db
from dimagi.utils.couch.pagination import DatatablesParams
from dimagi.utils.dates import DateSpan, force_to_datetime
from dimagi.utils.parsing import json_format_datetime, string_to_datetime
from dimagi.utils.timezones import utils as tz_utils
from dimagi.utils.web import json_request, get_url_base

DATE_FORMAT = "%Y-%m-%d"

class StandardHQReport(HQReport):
    user_filter = HQUserType.use_defaults()
    fields = ['corehq.apps.reports.fields.FilterUsersField']
    history = None
    use_json = False
    hide_filters = False
    custom_breadcrumbs = None

    def get_global_params(self):
        self.request_params = json_request(self.request.GET)
        # the history param lets you run a report as if it were a given time in the past
        hist_param = self.request_params.get('history', None)
        if hist_param:
            self.history = datetime.datetime.strptime(hist_param, DATE_FORMAT)
        self.user_filter, _ = FilterUsersField.get_user_filter(self.request)

        self.group = self.request_params.get('group','')
        self.individual = self.request_params.get('individual', '')
        self.case_type = self.request_params.get('case_type', '')

        self.users = util.get_all_users_by_domain(self.domain, self.group, self.individual, self.user_filter)

        if self.individual:
            self.name = "%s for %s" % (self.name, self.users[0].raw_username)
        self.get_parameters()
        self.context['report_hide_filters'] = self.hide_filters
        self.context['report_breadcrumbs'] = self.custom_breadcrumbs

    def get_parameters(self):
        # here's where you can define any extra parameters you want to process before
        # proceeding with the rest of get_report_context
        pass

    def get_report_context(self):
        self.context['standard_report'] = True
        self.context['layout_flush_content'] = True
        self.build_selector_form()
        self.context.update(util.report_context(self.domain,
                                        report_partial = self.report_partial,
                                        title = self.name,
                                        headers = self.headers,
                                        rows = self.rows,
                                        show_time_notice = self.show_time_notice
                                      ))
    
    def as_view(self):
        self.get_global_params()
        return super(StandardHQReport, self).as_view()

    def as_json(self):
        self.get_global_params()
        return super(StandardHQReport, self).as_json()


class StandardTabularHQReport(StandardHQReport):
    total_row = None
    default_rows = 10
    start_at_row = 0

#    exportable = True
    def get_headers(self):
        return DataTablesHeader()

    def get_rows(self):
        return self.rows

    def get_report_context(self):
        self.context['report_datatables'] = {
            "defaultNumRows": self.default_rows,
            "startAtRowNum": self.start_at_row
        }
        if self.use_json:
            self.context['ajax_source'] = reverse('json_report_dispatcher',
                                                  args=[self.domain, self.slug])
            self.context['ajax_params'] = [{'name': 'individual', 'value': self.individual},
                                            {'name': 'group', 'value': self.group},
                                            {'name': 'casetype', 'value': self.case_type},
                                            {'name': 'ufilter', 'value': [f.type for f in self.user_filter if f.show]}]
        else:
            self.context['ajax_source'] = False
            self.context['ajax_params'] = False

        self.headers = self.get_headers()
        self.rows = self.get_rows()

        self.context['header'] = self.headers
        self.context['rows'] = self.rows
        if self.total_row:
            self.context['total_row'] = self.total_row

        super(StandardTabularHQReport, self).get_report_context()


class PaginatedHistoryHQReport(StandardTabularHQReport):
    use_json = True
    fields = ['corehq.apps.reports.fields.FilterUsersField',
              'corehq.apps.reports.fields.SelectMobileWorkerField']

    count = 0
    _total_count = None
    
    @property
    def total_count(self):
        if self._total_count is not None:
            return self._total_count
        return self.count
    
    def get_parameters(self):
        self.userIDs = [user.user_id for user in self.users if user.user_id]
        self.usernames = dict([(user.user_id, user.username_in_report) for user in self.users])

    def json_data(self):
        params = DatatablesParams.from_request_dict(self.request.REQUEST)
        rows = self.paginate_rows(params.start, params.count)
        return {
            "sEcho": params.echo,
            "iTotalDisplayRecords": self.count,
            "iTotalRecords": self.total_count,
            "aaData": rows
        }

    def paginate_rows(self, skip, limit):
        # gather paginated rows here
        self.count = 0
        return []


class StandardDateHQReport(StandardHQReport):

    def __init__(self, domain, request, base_context=None):
        base_context = base_context or {}
        super(StandardDateHQReport, self).__init__(domain, request, base_context)
        self.datespan = self.get_default_datespan()
        self.datespan.is_default = True

    def get_default_datespan(self):
        return DateSpan.since(7, format="%Y-%m-%d", timezone=self.timezone)

    def get_global_params(self):
        if self.request.datespan.is_valid() and not self.request.datespan.is_default:
            self.datespan.enddate = self.request.datespan.enddate
{'fraction_non_alphanumeric': 0.08890832240198222, 'fraction_numerical': 0.001384637807899723, 'mean_word_length': 2.702914193200216, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2245492', 'n_tokens_mistral': 3907, 'n_tokens_neox': 3764, 'n_words': 845}